def augment_image(image_path, out_dir):
""" This function augments the passed in image with three
transformations. First it horizontally flips the image, then it rotates
it by 20 degrees, and finally, it horizontally flips it, rotates it by 5
degrees, and translates it on the y-axis by 50 units.
The three resultant images are saved to hf_{image_name}, rt_{image_name},
and tl_{image_name} in the provided directory, respectively."""
# Setup ImageDataGenerator, the normal parameters don't matter since we will manually
# apply transforms
image_path = os.path.abspath(image_path)
out_dir = os.path.abspath(out_dir)
image_filename_pattern = r'[^/]+$'
image_filename = re.search(image_filename_pattern, image_path)
if not image_filename:
print('Unable to extract filename from the provided path.')
sys.exit(1)
image_filename = image_filename.group(0)
datagen = ImageDataGenerator(rescale=1. / 255)
transform_params = {}
# Load the image using OpenCV2
img_array = cv2.imread(image_path, cv2.IMREAD_COLOR)
_, width, _ = img_array.shape
# Apply horizontal flip
transform_params['flip_horizontal'] = True
horizontal_image = datagen.apply_transform(img_array, transform_params)
transform_params.clear()
# Apply 20 degree rotation
transform_params['theta'] = 20.0
rotated_image = datagen.apply_transform(img_array, transform_params)
transform_params.clear()
# Apply translation
transform_params['flip_horizontal'] = True
transform_params['theta'] = 15.0
transform_params['ty'] = width * 0.2
translated_image = datagen.apply_transform(img_array, transform_params)
transform_params.clear()
# Save images
did_succeed = True
did_succeed = did_succeed and cv2.imwrite(
os.path.join(out_dir, image_filename), img_array)
did_succeed = did_succeed and cv2.imwrite(
os.path.join(out_dir, f'hf_{image_filename}'), horizontal_image)
did_succeed = did_succeed and cv2.imwrite(
os.path.join(out_dir, f'rt_{image_filename}'), rotated_image)
did_succeed = did_succeed and cv2.imwrite(
os.path.join(out_dir, f'tl_{image_filename}'), translated_image)
if did_succeed:
print(f'Successfully saved images to {out_dir}')
else:
print(f'Unable to save images to {out_dir}')
class LineDetectorModel(Model):
"""Model to detect lines of text in an image."""
def __init__(
self,
dataset_cls: type = IamParagraphsDataset,
network_fn: Callable = fcn,
dataset_args: Dict = None,
network_args: Dict = None,
):
"""Define the default dataset and network values for this model."""
super().__init__(dataset_cls, network_fn, dataset_args, network_args)
self.data_augmentor = ImageDataGenerator(**_DATA_AUGMENTATION_PARAMS)
self.batch_augment_fn = self.augment_batch
def loss(self): # pylint: disable=no-self-use
return "categorical_crossentropy"
def optimizer(self): # pylint: disable=no-self-use
return Adam(0.001 / 2)
def metrics(self): # pylint: disable=no-self-use
return None
def augment_batch(self, x_batch: np.ndarray,
y_batch: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""Perform different random transformations on the whole batch of x, y samples."""
x_augment, y_augment = zip(
*[self._augment_sample(x, y) for x, y in zip(x_batch, y_batch)])
return np.stack(x_augment, axis=0), np.stack(y_augment, axis=0)
def _augment_sample(self, x: np.ndarray,
y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""
Perform the same random image transformation on both x and y.
x is a 2d image of shape self.image_shape, but self.data_augmentor needs the channel image too.
"""
x_3d = np.expand_dims(x, axis=-1)
transform_parameters = self.data_augmentor.get_random_transform(
x_3d.shape)
x_augment = self.data_augmentor.apply_transform(
x_3d, transform_parameters)
y_augment = self.data_augmentor.apply_transform(
y, transform_parameters)
return np.squeeze(x_augment, axis=-1), y_augment
def predict_on_image(self, x: np.ndarray) -> np.ndarray:
"""Predict on a single input."""
return self.network.predict(np.expand_dims(x, axis=0))[0]
def evaluate(self,
x: np.ndarray,
y: np.ndarray,
batch_size: int = 32,
verbose: bool = False) -> float:
"""Evaluate the model."""
# pylint: disable=unused-argument
return self.network.evaluate(x, y, batch_size=batch_size)
class LineDetectModel(Model):
"""Model to detect lines of text in an image."""
def __init__(self,
network_fn: Callable = lenetFCN,
dataset: type = IAMPara):
"""Define the default dataset and network values for this model."""
super().__init__(network_fn, dataset)
print('[INFO] Arguments passed to data augmentation...',
DATA_AUGMENTATION_PARAMS)
self.data_augmentor = ImageDataGenerator(**DATA_AUGMENTATION_PARAMS)
self.batch_augment_fn = self.augment_batch
def metrics(self):
return None
def augment_batch(self, x_batch: np.ndarray,
y_batch: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""Performs different random transformations on the whole batch of x, y samples."""
x_augment, y_augment = zip(
*[self.augment_sample(x, y) for x, y in zip(x_batch, y_batch)])
return np.stack(x_augment, axis=0), np.stack(y_augment, axis=0)
def augment_sample(self, x: np.ndarray,
y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""
Perform the same random image transformation on both x and y.
x is a 2d image of shape self.image_shape, but self.data_augmentor needs the channel image too.
"""
x_3d = np.expand_dims(x, axis=-1)
transform_parameters = self.data_augmentor.get_random_transform(
x_3d.shape)
x_augment = self.data_augmentor.apply_transform(
x_3d, transform_parameters)
y_augment = self.data_augmentor.apply_transform(
y, transform_parameters)
return np.squeeze(x_augment, axis=-1), y_augment
def predict_on_image(self, x: np.ndarray) -> np.ndarray:
"""Returns the network predictions on x."""
return self.network.predict(np.expand_dims(x, axis=0))[0]
def expand_data(data, labels):
# Create data generator
transforms = {'tx': 0.1, 'ty': 0.1, 'flip_horizontal': True}
datagen = ImageDataGenerator()
new_data = []
new_labels = []
for d, l in zip(data, labels):
e = datagen.apply_transform(d, transforms)
new_data.append(e)
new_labels.append(l)
new_data = np.array(new_data)
new_labels = np.array(new_labels)
all_data = np.concatenate((data, new_data), axis=0)
all_labels = np.concatenate((labels, new_labels), axis=0)
return (all_data, all_labels)
class DataGenerator(keras.utils.Sequence):
def __init__(self,
list_IDs,
labels,
batch_size=32,
dim=(32, 32),
shuffle=True,
type_gen='train'):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.shuffle = shuffle
self.type_gen = type_gen
self.aug_gen = ImageDataGenerator()
print("all:", len(self.list_IDs), " batch per epoch",
int(np.floor(len(self.list_IDs) / self.batch_size)))
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
if self.type_gen == 'predict':
return X
else:
return X, y
def sequence_augment(self, img):
name_list = [
'rotate', 'width_shift', 'height_shift', 'brightness',
'flip_horizontal', 'width_zoom', 'height_zoom'
]
dictkey_list = [
'theta', 'ty', 'tx', 'brightness', 'flip_horizontal', 'zy', 'zx'
]
random_aug = np.random.randint(2, 5) # random 2-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list),
random_aug,
replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
dict_input['theta'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'ty': # width_shift
dict_input['ty'] = np.random.randint(-20, 20)
elif dictkey_list[i] == 'tx': # height_shift
dict_input['tx'] = np.random.randint(-20, 20)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.75, 1.25)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = bool(random.getrandbits(1))
elif dictkey_list[i] == 'zy': # width_zoom
dict_input['zy'] = np.random.uniform(0.75, 1.25)
elif dictkey_list[i] == 'zx': # height_zoom
dict_input['zx'] = np.random.uniform(0.75, 1.25)
img = self.aug_gen.apply_transform(img, dict_input)
return img
def albu_aug(self, image, tfms=albu_tfms):
seed = random.randint(0, 99999)
random.seed(seed)
image = tfms(image=image.astype('uint8'))['image']
return image
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X = [
np.empty((self.batch_size, self.dim[0], self.dim[1], 3)),
np.empty((self.batch_size, self.dim[0], self.dim[1], 3))
]
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
img = cv2.imread(ID)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (self.dim[1], self.dim[0]))
if self.type_gen == 'train':
img = self.sequence_augment(img)
new_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
new_img = np.expand_dims(new_img, -1)
new_img = automatedMSRCR(new_img, [10, 20, 30])
new_img = cv2.cvtColor(new_img[:, :, 0], cv2.COLOR_GRAY2RGB)
X[0][i] = img / 255.0
X[1][i] = new_img / 255.0
else:
new_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
new_img = np.expand_dims(new_img, -1)
new_img = automatedMSRCR(new_img, [10, 20, 30])
new_img = cv2.cvtColor(new_img[:, :, 0], cv2.COLOR_GRAY2RGB)
X[0][i] = img / 255.0
X[1][i] = new_img / 255.0
Y[i] = self.labels[ID]
return X, Y
class DataGen(keras.utils.Sequence):
def __init__(self, filepath, batch_size=4, target_size=(48,48)):
with open(filepath, 'r') as f:
lines = f.readlines()
self.target_size = target_size
self.data_dir = os.path.dirname(os.path.abspath(filepath))
self.batch_size = batch_size
self.class_num = 6
self.fnames = [line.replace('\n', '') for line in lines]
self.shuffle = True
self.aug_gen = ImageDataGenerator()
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.fnames) / self.batch_size))
def name(self):
return 'human'
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
# Find list of IDs
fnames = [self.fnames[k] for k in indexes]
# print(fnames)
# for fname in fnames:
# imgs = cv2.imread(os.path.join(self.data_dir, fname))
data = np.array([img_to_array(load_img(os.path.join(self.data_dir, fname.replace('pick','hat')), target_size=self.target_size))
for fname in fnames
]).astype('float32')
data = data/255.
y_data = np.array([img_to_array(load_img(os.path.join(self.data_dir, fname), target_size=self.target_size))
for fname in fnames
]).astype('float32')
y_data = y_data/255.
# data = self.img_augment(data)
# y_data = data.copy()
# data_len = y_data.shape[0]
# w,h=self.target_size
# for i in range(data_len):
# y_data[i][0:10,0:w,0:3] = (0,0,0)
# y_data[i][h-10:h,0:w,0:3] = (0,0,0)
# y_data[i][0:h,0:10,0:3] = (0,0,0)
# y_data[i][0:h,w-10:w,0:3] = (0,0,0)
return data, y_data
def img_augment(self, data):
name_list = ['rotate','width_shift','height_shift',
'brightness','flip_horizontal','width_zoom',
'height_zoom']
dictkey_list = ['theta','ty','tx',
'brightness','flip_horizontal','zy',
'zx']
# dictkey_list = ['ty','tx','zy','zx']
random_aug = np.random.randint(2, 5) # random 2-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list), random_aug, replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
dict_input['theta'] = np.random.randint(-10, 10)
# elif dictkey_list[i] == 'ty': # width_shift
# dict_input['ty'] = np.random.randint(-10, 10)
# elif dictkey_list[i] == 'tx': # height_shift
# dict_input['tx'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.15,1)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = True
elif dictkey_list[i] == 'zy': # width_zoom
dict_input['zy'] = np.random.uniform(0.5,1.5)
elif dictkey_list[i] == 'zx': # height_zoom
dict_input['zx'] = np.random.uniform(0.5,1.5)
data_len = data.shape[0]
for i in range(data_len):
data[i] = self.aug_gen.apply_transform(data[i], dict_input)/255.0
return data
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.fnames))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def augment_dataset(samples: list, augmented_dir:str, multiplier:int=2, register:bool=False, res:tuple=(120, 160)):
"""
Augments a dataset. Will attempt to balance classes.
Params
------
samples: list
List of sample annotation tuples (rgb path, lwir path, class label)
augmented_dir: str
Directory to store augmented data at
multiplier: int
Multiplication factor for number of new samples
register: bool
Whether to align RGB images with LWIR images
Returns
------
A list containing tuples of (rgb path, lwir path, class name) for new augmented dataset
"""
if os.path.exists(augmented_dir):
shutil.rmtree(augmented_dir)
datagen = ImageDataGenerator(
horizontal_flip=True,
rotation_range=30,
zoom_range=[0.5, 1],
)
classes = {}
# Get classes
for sample in samples:
classes[sample[-1]] = classes.get(sample[-1], 0) + 1
# Get maximum class samples
max_class = max(classes.items(), key=lambda x: x[1])
class_multipliers = {x: max_class[1] * multiplier // y for x, y in classes.items()}
class_remainders = {x: (max_class[1] * multiplier) % y for x, y in classes.items()}
# Generate output directory structure
for label in classes:
os.makedirs(os.path.join(augmented_dir, label, f"{label}_single_1", "lwir"))
os.makedirs(os.path.join(augmented_dir, label, f"{label}_single_1", "rgb"))
# Augment and copy
augmented_samples = []
for rgb_path, lwir_path, label in tqdm(samples, position=0):
rgb = cv2.imread(rgb_path)
if register:
rgb = cv2.resize(rgb, tuple(cfg.large.res))
rgb = cv2.warpAffine(rgb, cfg.large.matrix, tuple(cfg.large.res))
rgb = cv2.resize(rgb, res)
lwir = cv2.imread(lwir_path)
lwir = cv2.resize(lwir, res)
name = (rgb_path.split("/")[-1]).split(".")[0][4:]
out_path = os.path.join(augmented_dir, label, f"{label}_single_1")
# Copy original
augmented_samples.append(save(out_path, name, rgb, lwir, label, extension=""))
# Flipped
if multiplier > 1:
rgb_t = datagen.apply_transform(rgb, {"flip_horizontal": True})
lwir_t = datagen.apply_transform(lwir, {"flip_horizontal": True})
augmented_samples.append(save(out_path, name, rgb_t, lwir_t, label, extension=0))
# Random transformations
remainder = 1 if class_remainders[label] > 0 else 0
class_remainders[label] -= 1
for i in range(2, class_multipliers[label] + remainder):
trans = datagen.get_random_transform(rgb.shape)
rgb_t = datagen.apply_transform(rgb, trans)
lwir_t = datagen.apply_transform(lwir, trans)
augmented_samples.append(save(out_path, name, rgb_t, lwir_t, label, extension=i))
return augmented_samples
class Generate(Sequence, Prep): # Keras generator class
def __init__(self, root, trainingmode, augmentation, nb_classes, batchsize,
rotation, translation_xy, scale_xy, flip_h, flip_v):
Prep.__init__(self, root)
self.train, self.validate = Prep.initializedatalist(self)
self.trainingmode = trainingmode
self.augmentation = augmentation
self.nb_classes = nb_classes
self.batchsize = batchsize
self.rotation = rotation
self.translation_xy = translation_xy
self.scale_xy = scale_xy
self.flip_h = flip_h
self.flip_v = flip_v
self.datagenI = ImageDataGenerator(fill_mode = "constant", cval = -1000)
self.datagenL = ImageDataGenerator(fill_mode = "constant", cval = 0)
with open (os.path.join(root, "0_sorted", "att_clswt.dat"), "rb") as outfile:
self.clswt = pickle.load(outfile)
def wImg(self, label): # Computes the weight matrix
wImg = np.zeros(label.shape)
for k in range(self.nb_classes):
ind = np.where(label == k)
wImg[ind] = self.clswt[k]
return wImg
def __len__(self):
if self.trainingmode:
return int(np.ceil(len(self.train)/self.batchsize))
else:
return int(np.ceil(len(self.validate)/self.batchsize))
def __getitem__(self, counter):
### TRAINING GENERATOR ###
if self.trainingmode:
imagebatch = np.zeros((0,512,512,1))
labelbatch = np.zeros((0,262144,self.nb_classes))
weightmatrixbatch = np.zeros((0,262144))
scans = self.train[counter*self.batchsize:(counter+1)*self.batchsize]
for each in scans:
with open (each, "rb") as outfile:
data = pickle.load(outfile)
image, label = data[0], data[1]
image, label = image.astype(np.float32), label.astype(np.float32)
weightmatrix = self.wImg(label).flatten()
weightmatrix = np.expand_dims(weightmatrix, axis = 0)
label = to_categorical(label, self.nb_classes)
# data augmentation
flip_h, flip_v = 0, 0
if self.flip_h:
flip_h = np.random.randint(2)
if self.flip_v:
flip_v = np.random.randint(2)
parameters = {"theta": np.random.randint(-self.rotation, self.rotation+1),
"tx": np.random.randint(-self.translation_xy, self.translation_xy+1),
"ty": np.random.randint(-self.translation_xy, self.translation_xy+1),
"shear": 0,
"zx": 1 + np.random.uniform(-self.scale_xy, self.scale_xy),
"zy": 1 + np.random.uniform(-self.scale_xy, self.scale_xy),
"flip_horizontal": flip_h, "flip_vertical": flip_v}
image, label = self.datagenI.apply_transform(image, parameters), self.datagenL.apply_transform(label, parameters)
image, label = image.astype(np.float32), label.astype(np.float32)
image = np.expand_dims(image, axis = 0)
label = np.expand_dims(label, axis = 0)
label = label.reshape(1,262144,self.nb_classes)
imagebatch = np.concatenate((imagebatch, image), axis=0)
labelbatch = np.concatenate((labelbatch, label), axis=0)
weightmatrixbatch = np.concatenate((weightmatrixbatch, weightmatrix), axis=0)
return (imagebatch, labelbatch, weightmatrixbatch)
### VALIDATION GENERATOR ###
else:
imagebatch = np.zeros((0,512,512,1))
labelbatch = np.zeros((0,262144,self.nb_classes))
scans = self.validate[counter*self.batchsize:(counter+1)*self.batchsize]
for each in scans:
with open (each, "rb") as outfile:
data = pickle.load(outfile)
image, label = data[0], data[1]
image, label = image.astype(np.float32), label.astype(np.float32)
label = to_categorical(label, self.nb_classes)
label = label.reshape(262144,self.nb_classes)
image = np.expand_dims(image, axis = 0)
label = np.expand_dims(label, axis = 0)
imagebatch = np.concatenate((imagebatch, image), axis=0)
labelbatch = np.concatenate((labelbatch, label), axis=0)
return (imagebatch, labelbatch)
class TextImageGenerator:
def __init__(self,
img_dirpath,
labels_path,
img_w,
img_h,
batch_size,
downsample_factor,
idxs,
max_text_len=70,
training=True):
self.img_h = img_h
self.img_w = img_w
self.batch_size = batch_size
self.max_text_len = max_text_len
self.training = training
self.idxs = idxs
self.downsample_factor = downsample_factor
self.img_dirpath = img_dirpath # image dir path
self.labels = json.load(
open(labels_path,
'rb'), encoding="utf8") if labels_path != None else None
self.img_dir = []
for img_file in os.listdir(self.img_dirpath): # images list
if img_file != 'labels.json' and '.ipynb' not in img_file:
self.img_dir.append(img_file)
random.shuffle(self.img_dir)
if self.idxs is not None:
self.img_dir = [self.img_dir[idx] for idx in self.idxs]
self.n = len(self.img_dir) # number of images
self.indexes = list(range(self.n))
self.cur_index = 0
self.imgs = np.ones((self.n, self.img_h, self.img_w, 3),
dtype=np.float16)
self.texts = []
image_datagen_args = {
'shear_range': 0.1,
'zoom_range': 0.01,
'width_shift_range': 0.001,
'height_shift_range': 0.1,
'rotation_range': 1,
'horizontal_flip': False,
'vertical_flip': False
}
self.image_datagen = ImageDataGenerator(**image_datagen_args)
def build_data(self):
print(self.n, " Image Loading start... ", self.img_dirpath)
for i, img_file in enumerate(self.img_dir):
img = image.load_img(self.img_dirpath + img_file,
target_size=SIZE[::-1],
interpolation='bicubic')
img = image.img_to_array(img)
img = preprocess_input(img)
self.imgs[i] = img
if self.labels != None:
self.texts.append(self.labels[img_file][:MAX_LEN])
else:
#valid mode
self.texts.append(img_file)
print("Image Loading finish...")
def next_sample(self):
self.cur_index += 1
if self.cur_index >= self.n:
self.cur_index = 0
random.shuffle(self.indexes)
return self.imgs[self.indexes[self.cur_index]].astype(
np.float32), self.texts[self.indexes[self.cur_index]]
def next_batch(self):
while True:
X_data = np.zeros([self.batch_size, self.img_w, self.img_h, 3],
dtype=np.float32) # (bs,2048, 64, 3)
Y_data = np.zeros([self.batch_size, self.max_text_len],
dtype=np.float32) # (bs, 70)
input_length = np.ones((self.batch_size, 1), dtype=np.float32) * (
self.img_w // self.downsample_factor - 2) # (bs, 1)
label_length = np.zeros((self.batch_size, 1),
dtype=np.float32) # (bs, 1)
for i in range(self.batch_size):
img, text = self.next_sample()
if self.training:
params = self.image_datagen.get_random_transform(img.shape)
img = self.image_datagen.apply_transform(img, params)
img = img.transpose((1, 0, 2))
X_data[i] = img
Y_data[i, :len(text)] = text_to_labels(text)
label_length[i] = len(text)
inputs = {
'the_inputs': X_data, # (bs,2048, 64, 1)
'the_labels': Y_data, # (bs,70)
'input_length': input_length, # (bs, 1)
'label_length': label_length # (bs, 1)
}
outputs = {'ctc': np.zeros([self.batch_size])} # (bs, 1)
yield (inputs, outputs)
def AugTrain_reg(in_vol_in, in_vol_out, num_out, minmax):
new_vol_in = np.empty(
[in_vol_in.shape[0], in_vol_in.shape[1], in_vol_in.shape[2], 1])
new_vol_out = np.zeros((num_out, 1))
print(in_vol_in.shape)
sh = new_vol_in.shape
Vols = [[], []]
datagen = ImageDataGenerator()
r_list = [-0.5, 0.25, 0.11]
flip_list = [False, False, True]
for i in range(len(r_list)):
# prepare i different transformations
r = r_list[i]
flip = flip_list[i]
# print("r= ", r)
imageData = np.empty([sh[0], sh[1], sh[2]])
image = np.zeros([sh[0], sh[1], 1])
#print("Shape before augmentation: " + str(type(new_vol_in[1][1][45][0])))
for j in range(sh[2]):
# apply transformation on image
image1 = np.expand_dims(in_vol_in[:, :, j], 2)
#print(image1)
image = datagen.apply_transform(
image1,
transform_parameters={
'flip_vertical': flip,
# vertical flip results in upside down image
'tx': r * 20, # vertical translation
'ty': r * 10
}) # horizontal translation
# Do preprocess on input images and change gray level of images to [0 255] and eliminate nan and -inf values from image pixles value and active below process
image = np.squeeze(image, axis=2)
image = np.uint8(image)
#print(image)
image[image < 0] = 0
image[image > 255] = 255
image = image / 255
#print("Val " + str(image[64,64]))
#plt.imshow(image[:, :])
#plt.show()
#print("IMAGE " + str(image))
#print("VAL: " + str(image[1][1]))
imageData[:, :, j] = image # (128,128,99)
#plt.imshow(imageData[:, :, 45])
#plt.show()
imageData = imageData.reshape(sh[0], sh[1], sh[2], 1)
Vols[0].append(imageData)
# Add orgilal image in Vols[0]/ After preprocess of input images , active below process
in_vol_in = np.uint8(in_vol_in)
#in_vol_in = in_vol_in / 255
in_vol_in[in_vol_in < 0] = 0
in_vol_in[in_vol_in > 255] = 255
in_vol_in = in_vol_in.reshape(sh[0], sh[1], sh[2], 1)
Vols[0].append(in_vol_in)
#print("Shape after augmentation: " + str(type(in_vol_in[1][1][45][0])))
# Normalized coeff with total min and total max of all coefficients
#print("voor"+ str(in_vol_out))
new_vol_out = (in_vol_out - minmax[0]) / (minmax[1] - minmax[0])
#print("na"+ str(new_vol_out))
Vols[1].append(new_vol_out[:, 1])
Vols[1].append(new_vol_out[:, 1])
Vols[1].append(new_vol_out[:, 1])
Vols[1].append(new_vol_out[:, 1])
# Vols.append(new_vol_in)
# Vols.append(new_vol_out)
return Vols
class DataGen(Sequence):
def __init__(self,
filepath,
batch_size=8,
class_num=2,
target_size=(224, 224),
n_channels=3,
preprocess_input=None,
with_aug=True,
shuffle=True,
path_dataset=None,
type_gen='train',
option=None):
data_dict = readfile_to_dict(filepath)
data_keys = list(data_dict.keys())
self.shuffle = shuffle
self.batch_size = batch_size
self.labels = data_dict
self.list_IDs = data_keys
self.target_size = target_size
self.path_dataset = path_dataset
self.type_gen = type_gen
self.option = option
self.n_channels = n_channels
self.class_num = class_num
self.preprocess_input = preprocess_input
self.aug_gen = ImageDataGenerator()
self.steps_per_epoch = len(self.list_IDs) // self.batch_size
print("all:", len(self.list_IDs), " batch per epoch",
self.steps_per_epoch)
ww, hh = 704, np.floor(576 * 0.8)
x0, y0 = 0, int(np.floor(576 * 0.1))
# ww,hh=target_size[0]+19+x0,target_size[1]+19+y0
step = 70
self.pos = []
while True:
if y0 + target_size[1] > hh:
break
x0 = 0
while True:
if x0 + target_size[0] > ww:
break
self.pos.append((x0, y0))
x0 += step
y0 += step
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
# print(int(np.floor(len(self.list_IDs) / self.batch_size)), len(self.pos))
return int(np.floor(len(self.list_IDs) / self.batch_size)) * len(
self.pos)
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
'Generate one batch of data'
index_i = index // len(self.pos)
pos_i = index % len(self.pos)
# Generate indexes of the batch
indexes = self.indexes[index_i * self.batch_size:(index_i + 1) *
self.batch_size]
# Find list of IDs
ids = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(ids, self.pos[pos_i])
if self.type_gen == 'predict':
return X
else:
return X, y
def sequence_augment(self, sequence):
name_list = [
'rotate', 'width_shift', 'height_shift', 'brightness',
'flip_horizontal', 'width_zoom', 'height_zoom'
]
dictkey_list = [
'theta', 'ty', 'tx', 'brightness', 'flip_horizontal', 'zy', 'zx'
]
# dictkey_list = ['ty','tx','zy','zx']
random_aug = np.random.randint(2, 5) # random 2-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list),
random_aug,
replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
dict_input['theta'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'ty': # width_shift
dict_input['ty'] = np.random.randint(-5, 5)
elif dictkey_list[i] == 'tx': # height_shift
dict_input['tx'] = np.random.randint(-5, 5)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.15, 1)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = True
elif dictkey_list[i] == 'zy': # width_zoom
dict_input['zy'] = np.random.uniform(0.5, 1.5)
elif dictkey_list[i] == 'zx': # height_zoom
dict_input['zx'] = np.random.uniform(0.5, 1.5)
len_seq = sequence.shape[0]
for i in range(len_seq):
sequence[i] = self.aug_gen.apply_transform(sequence[i], dict_input)
return sequence
def __data_generation(self, ids, pos):
'Generates data containing batch_size samples'
# Initialization
Y = np.zeros((self.batch_size, self.class_num))
X = []
x = None
y = None
for i, ID in enumerate(ids): # ID is name of file
path_file = os.path.join(self.path_dataset, ID)
# print(path_file)
img = cv2.imread(path_file)
# 取图像中间的0.8部分
hh, ww, _ = img.shape
xx, yy = int(0), int(hh * 0.1)
ww, hh = int(ww), int(hh * 0.8)
img = img[yy:yy + hh, xx:xx + ww]
img = img[:, :, [2, 1, 0]] # to RGB
# if self.preprocess_input is not None:
# img = self.preprocess_input(img)
if i < len(ids) / 2:
x, y = pos[0], pos[1]
else:
# 随机切一块
h, w, n = img.shape
# print(img.shape)
x = np.random.randint(0, w - self.target_size[0])
y = np.random.randint(0, h - self.target_size[1])
(w, h) = self.target_size
# print(x,y,w,h)
X.append(img[y:y + h, x:x + w])
# print(x,y,w,h)
X = np.array(X)
if self.type_gen == 'train':
X = self.sequence_augment(X) # apply the same rule
else:
X = X
# X = X/255.
X = X / 127.5
X = X - 1.
return X, Y
class DataGenerator_segmentation(Sequence):
def __init__(self,
list_IDs,
labels,
batch_size=8,
n_classes=3,
shuffle=True,
validation=False):
'Initialization'
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
if validation:
self.classe_weight = {0: 1.66666667, 1: 1.19047619, 2: 0.64102564}
data_gen_args = dict()
data_gen_args2 = dict()
else:
self.classe_weight = {0: 1.78253119, 1: 2.62467192, 2: 0.48590865}
data_gen_args = dict(
horizontal_flip=True,
vertical_flip=True,
)
data_gen_args2 = dict(
horizontal_flip=True, #rescale=1./255,
vertical_flip=True,
)
self.image_datagen = ImageDataGenerator(**data_gen_args)
self.mask_datagen = ImageDataGenerator(**data_gen_args2)
seed = 1
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
DATAPATH = 'ISIC2017/Tudojunto_r/imagens/'
DATAPATH2 = 'ISIC2017/Tudojunto_r/segm/'
#DATAPATH='ISIC2017/Tudojunto/imagens/'
#DATAPATH2='ISIC2017/Tudojunto/segm/'
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, 572, 572, 3))
ys = np.empty((self.batch_size, 388, 388, 1))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Generate Transform
transform_i = self.image_datagen.get_random_transform(
(572, 572, 3), seed=1)
transform_s = self.mask_datagen.get_random_transform((388, 388, 1),
seed=1)
# Store sample
img = imread(DATAPATH + ID + '.jpg')
img = img / 255
mask = np.expand_dims(imread(DATAPATH2 + ID + '.png'), axis=-1)
mask = mask / 255
X[i, ] = self.image_datagen.apply_transform(
img, transform_parameters=transform_i).astype(np.float32)
ys[i, ] = self.mask_datagen.apply_transform(
mask, transform_parameters=transform_s).astype(np.float32)
return X, ys
class DataGenerator(Sequence):
def __init__(self,
list_IDs,
labels,
batch_size=8,
n_classes=3,
shuffle=True,
validation=False):
'Initialization'
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
if validation:
self.classe_weight = {0: 1.66666667, 1: 1.19047619, 2: 0.64102564}
data_gen_args = dict()
data_gen_args2 = dict()
else:
self.classe_weight = {0: 1.78253119, 1: 2.62467192, 2: 0.48590865}
data_gen_args = dict(
horizontal_flip=True,
vertical_flip=True,
)
data_gen_args2 = dict(
horizontal_flip=True, #rescale=1./255,
vertical_flip=True,
)
self.image_datagen = ImageDataGenerator(**data_gen_args)
self.mask_datagen = ImageDataGenerator(**data_gen_args2)
seed = 1
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y, sample_weight = self.__data_generation(list_IDs_temp)
return X, y, sample_weight
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
DATAPATH = 'ISIC2017/Tudojunto_r/imagens/'
DATAPATH2 = 'ISIC2017/Tudojunto_r/segm/'
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, 572, 572, 3))
#ys = np.empty((self.batch_size,388, 388, 1 ))
ys = np.empty((self.batch_size, 150544, 1))
yc = np.empty((self.batch_size, 3))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Generate Transform
transform_i = self.image_datagen.get_random_transform(
(572, 572, 3), seed=1)
#print(transform_i)
transform_s = self.mask_datagen.get_random_transform((388, 388, 1),
seed=1)
#print(transform_s)
# Store sample
img = imread(DATAPATH + ID + '.jpg')
img = img / 255
#print(np.amax(img))
mask = np.expand_dims(imread(DATAPATH2 + ID + '.png'), axis=-1)
mask = mask / 255
#img2 = self.image_datagen.apply_transform(img , transform_parameters=transform_i)
#print(np.amax(img2))
X[i, ] = self.image_datagen.apply_transform(
img, transform_parameters=transform_i).astype(np.float32)
#ys[i,] = self.mask_datagen.apply_transform(mask , transform_parameters=transform_s)
mask2 = self.mask_datagen.apply_transform(
mask, transform_parameters=transform_s).astype(np.float32)
ys[i, ] = np.reshape(mask2, (150544, 1))
#print((np.reshape(ys[i,], (150544,1))).shape)
#imshow(img)
#plt.show()
#imshow(img2)
#plt.show()
#print(img.shape)
#X[i,] = img
#ys[i,] = np.expand_dims(imread(DATAPATH2 + ID + '.png'), axis=-1)
# Store class
yc[i] = to_categorical(self.labels[ID], 3)
# Generate Weights
rounded_train = np.argmax(yc, axis=1)
#print("rounded_train:",rounded_train.shape,rounded_train)
#print("yc",yc)
weight_class = np.array(
[self.classe_weight[cls] for cls in rounded_train])
#print("weight_class",weight_class)
#mask_ones = np.ones(388,388,1)
#weight_segm = np.ones(((self.batch_size),388,388))
#tf.expand_dims(weight_mask,axis=0)
#weight_segm = np.ones(((self.batch_size),388,388,1))
#weight_segm = tf.expand_dims(weight_segm,axis=0)
#weight_segm = np.ones(((self.batch_size),1))
# weight_segm = np.array([self.classe_weight[cls] for cls in rounded_train])
weight_segm = np.ones(((self.batch_size), 150544))
return X, [yc, ys], [weight_class, weight_segm]
class TripletGenerator(Sequence):
def __init__(self,
file_path,
image_path,
image_test_path,
nb_classes_batch,
nb_images_per_class_batch,
target_size=TARGET_SIZE,
subset='training',
validation_split=0.0,
seed=42,
is_reptile=False,
shuffle=True,
excluded_classes=['new_whale'],
class_weight_type=None,
**kwargs):
self.file_path = file_path
self.shuffle = shuffle
self.target_size = target_size
self.nb_classes_batch = nb_classes_batch
self.nb_images_per_class_batch = nb_images_per_class_batch
self.batch_size = nb_classes_batch * nb_images_per_class_batch
self.image_path = image_path
self.image_test_path = image_test_path
self.df = pd.read_csv(file_path)
self.index_array = None
self.lock = threading.Lock()
self.is_reptile = is_reptile
logger.info('exclude_class: {}'.format(excluded_classes))
logger.info('class_weight_type: {}'.format(class_weight_type))
blacklisted_class_ix = self.df['Id'].isin(excluded_classes)
logger.info("{} instances excluded".format(
np.sum(blacklisted_class_ix)))
df = self.df[~blacklisted_class_ix]
classes = list(df['Id'].unique())
self.class_indices = dict(zip(classes, range(len(classes))))
self.class_inv_indices = dict(zip(range(len(classes)), classes))
train_classes, test_classes = train_test_split(
classes, test_size=validation_split, random_state=seed)
if subset == 'training':
self.df = df[df['Id'].isin(train_classes)]
else:
self.df = df[df['Id'].isin(test_classes)]
logger.info("data has shape: {}".format(self.df.shape))
classes = self.df['Id'].values
self.classes = np.array([self.class_indices[cls] for cls in classes])
self.filenames = np.array(self.df['Image'])
self.class_indices_subset = {
k: v
for k, v in self.class_inv_indices.items() if k in self.classes
}
kwargs.update({'preprocessing_function': _preprocess_input})
logger.info(kwargs)
self.image_generator = ImageDataGenerator(**kwargs)
# test time generator
self.image_generator_inference = ImageDataGenerator(
preprocessing_function=_preprocess_input)
def get_test_images_and_names(self):
img_names = os.listdir(self.image_test_path)
filepaths = [
os.path.join(self.image_test_path, img_name)
for img_name in img_names
]
_x = np.zeros((len(filepaths), ) + self.target_size, dtype='float32')
for i, _path in enumerate(filepaths):
_img = load_img(_path, target_size=self.target_size)
_x[i] = img_to_array(_img)
if hasattr(_img, 'close'):
_img.close()
return _x, img_names
def __len__(self):
"""
number of steps per epoch
number of classes in subet / nb_classes_batch
:return:
"""
return len(self.class_indices_subset) // self.nb_classes_batch
def get_nb_classes(self):
"""
number of all classes
:return:
"""
return len(self.class_indices)
def _set_index_array(self):
self.index_array = list(self.class_indices_subset.keys())
if self.shuffle:
self.index_array = np.random.permutation(self.index_array)
def on_epoch_end(self):
"""
shuffle at epoch end
"""
self._set_index_array()
def get_train_image_from_class_ix(self, ixs):
return self._get_batches_of_transformed_samples(ixs)
def __getitem__(self, idx):
if self.index_array is None:
self._set_index_array()
# unique classes to pull
index_array = self.index_array[idx * self.nb_classes_batch:self.
nb_classes_batch * (idx + 1)]
return self._get_batches_of_transformed_samples(index_array)
def _get_batches_of_transformed_samples(self, index_array):
"""
validation and training behaves the same way w.r.t. augmentation
:param index_array:
:return:
"""
batch_x = []
for i, class_ix in enumerate(index_array):
_samples = np.zeros(tuple([self.nb_images_per_class_batch] +
list(self.target_size)),
dtype='float32')
filenames = self.filenames[self.classes == class_ix]
if len(filenames) > self.nb_images_per_class_batch:
np.random.shuffle(filenames)
filenames = filenames[:self.nb_images_per_class_batch]
logger.debug("{} files for class {}".format(
len(filenames), class_ix))
for j, filename in enumerate(filenames):
img = load_img(os.path.join(self.image_path, filename),
target_size=self.target_size)
x = img_to_array(img, data_format='channels_last')
# Pillow images should be closed after `load_img`,
# but not PIL images.
if hasattr(img, 'close'):
img.close()
_samples[j] = self.image_generator.standardize(x)
# require at least `nb_images_per_class_batch` per class
nb_missing = _samples.shape[0] - j - 1
# select images to transform. No need to standardize again.
img_ix_transformed = np.random.choice(j + 1, nb_missing)
for img_ix, k in zip(img_ix_transformed, range(nb_missing)):
x = _samples[img_ix]
params = self.image_generator.get_random_transform(
self.target_size)
x = self.image_generator.apply_transform(x, params)
_samples[j + k + 1] = x
logger.debug("{} transformations for class {}".format(
nb_missing, class_ix))
batch_x.append(_samples)
batch_x = np.vstack(batch_x)
# build batch of labels
_labels = range(10) if self.is_reptile else index_array
batch_y = np.repeat(_labels, self.nb_images_per_class_batch)
if self.is_reptile:
batch_y = to_categorical(batch_y, num_classes=10)
assert len(batch_y), len(batch_x)
return batch_x, batch_y
def get_test_generator(self, x):
"""
:param x: test data
:return: NumpyArrayIterator
"""
return self.image_generator_inference.flow(x=x,
shuffle=False,
batch_size=self.batch_size)
class DataGenerator(Sequence):
'Generates data for Keras'
def __init__(self,
list_IDs,
labels,
batch_size=32,
dim=(32, 32),
n_channels=1,
n_sequence=4,
shuffle=True,
path_dataset=None,
type_gen='train',
option=None):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_sequence = n_sequence + 1 # get n_sequence diff image
self.shuffle = shuffle
self.path_dataset = path_dataset
self.type_gen = type_gen
self.option = option
self.aug_gen = ImageDataGenerator()
print("all:", len(self.list_IDs), " batch per epoch",
int(np.floor(len(self.list_IDs) / self.batch_size)))
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
D, X, y = self.__data_generation(list_IDs_temp)
if self.type_gen == 'predict':
return {'input_1': D, 'input_2': X}
elif self.type_gen == 'train' or self.type_gen == 'test':
return ({'input_1': D, 'input_2': X}, y)
elif self.type_gen == 'quantize':
return np.concatenate((D, X), axis=3), y
def get_sampling_frame(self, len_frames):
'''
Sampling n_sequence frame from video file
Input:
len_frames -- number of frames that this video have
Output:
index_sampling -- n_sequence frame indexs from sampling algorithm
'''
# Define maximum sampling rate
random_sample_range = 9
if random_sample_range * self.n_sequence > len_frames:
random_sample_range = len_frames // self.n_sequence
# Randomly choose sample interval and start frame
sample_interval = np.random.randint(3, random_sample_range + 1)
start_i = np.random.randint(
0, len_frames - sample_interval * self.n_sequence + 1)
# Get n_sequence index of frames
index_sampling = []
end_i = sample_interval * self.n_sequence + start_i
for i in range(start_i, end_i, sample_interval):
if len(index_sampling) < self.n_sequence:
index_sampling.append(i)
return index_sampling
def sequence_augment(self, sequence, rgb_img=None):
name_list = [
'rotate', 'width_shift', 'height_shift', 'brightness',
'flip_horizontal', 'width_zoom', 'height_zoom'
]
dictkey_list = [
'theta', 'ty', 'tx', 'brightness', 'flip_horizontal', 'zy', 'zx'
]
# dictkey_list = ['ty','tx','zy','zx']
random_aug = np.random.randint(2, 5) # random 2-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list),
random_aug,
replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
dict_input['theta'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'ty': # width_shift
dict_input['ty'] = np.random.randint(-60, 60)
elif dictkey_list[i] == 'tx': # height_shift
dict_input['tx'] = np.random.randint(-30, 30)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.15, 1)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = True
elif dictkey_list[i] == 'zy': # width_zoom
dict_input['zy'] = np.random.uniform(0.5, 1.5)
elif dictkey_list[i] == 'zx': # height_zoom
dict_input['zx'] = np.random.uniform(0.5, 1.5)
len_seq = sequence.shape[0]
for i in range(len_seq):
sequence[i] = self.aug_gen.apply_transform(sequence[i], dict_input)
if rgb_img is not None:
rgb_img = self.aug_gen.apply_transform(rgb_img, dict_input)
return sequence
def __data_generation2(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
D = np.empty((self.batch_size, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
X = np.empty((self.batch_size, self.n_sequence,
*self.dim)) # X : (n_samples, n_sequence, *dim)
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = self.path_dataset + ID + '.mp4'
cap = cv2.VideoCapture(path_file)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
new_image = None
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
gray = cv2.cvtColor(new_image, cv2.COLOR_BGR2GRAY)
X[i, j, :, :] = gray
D[i, :, :, :] = new_image
if self.type_gen == 'train':
# X[i,] = self.sequence_augment(X[i,])/255.0
X[i, ] = X[i, ] / 255.0
D[i, ] = D[i, ] / 255.0
else:
X[i, ] = X[i, ] / 255.0
D[i, ] = D[i, ] / 255.0
if self.option == 'RGBdiff':
X[i, ] = calculateRGBdiff(X[i, ])
# X = np.absolute(X)
# X[i,] = np.delete(X[i,], 0, axis=0) # remove first image
Y[i] = self.labels[ID]
cap.release()
X = np.rollaxis(X, 1, 4)
return D, X[:, :, :, 1:self.n_sequence], Y # remove first image
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X = np.empty((self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
Y = np.empty((self.batch_size), dtype=int)
D = np.empty((self.batch_size, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
S = np.empty((self.batch_size, self.n_sequence,
*self.dim)) # X : (n_samples, n_sequence, *dim)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = os.path.join(self.path_dataset, ID + '.mp4')
cap = cv2.VideoCapture(path_file)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
X[i, j, :, :, :] = new_image
if self.type_gen == 'train':
X[i, ] = self.sequence_augment(X[i, ]) # apply the same rule
else:
X[i, ] = X[i, ]
n = len(X[i, ])
# always get the last image
D[i, :, :, :] = X[i, n - 1, :, :, :]
# convert to gray image and normalize to 0~1
for k in range(n):
gray = X[i, k, :, :]
gray = np.array(gray, dtype=np.float32)
gray = cv2.cvtColor(gray, cv2.COLOR_BGR2GRAY)
S[i, k, :, :] = gray / 255.0
D[i, :, :, :] = D[i, :, :, :] / 255.0
if self.option == 'RGBdiff':
S[i, ] = calculateRGBdiff(S[i, ])
Y[i] = self.labels[ID]
cap.release()
S = np.rollaxis(S, 1, 4)
return D, S[:, :, :, 1:self.n_sequence], Y # remove first image
class VideoFrameGenerator(keras.utils.Sequence):
def __init__(
self,
list_IDs,
labels,
batch_size=32,
dim=(32, 32),
n_channels=3,
n_sequence=10,
shuffle=True,
type_gen="train",
):
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_sequence = n_sequence # number of frames to extract
self.shuffle = shuffle
self.type_gen = type_gen
self.sampl_mode = "2"
self.aug_gen = ImageDataGenerator()
print(f"Videos: {len(self.list_IDs)}, batches per epoch: "
f"{int(np.floor(len(self.list_IDs) / self.batch_size))}")
self.on_epoch_end()
def __len__(self):
# number of batches per epoch
return int(np.floor(len(self.list_IDs) / self.batch_size))
def on_epoch_end(self):
# updating index after epoch
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
return X, y
def frame_sampling(self, len_frames):
# create a list of frames
frames = list(range(len_frames))
# sampling choice
if self.sampl_mode == "1":
# create chunks
chunks = list(self.get_chunks(frames, self.n_sequence))
sampling = self.sampling_mode_1(chunks)
elif self.sampl_mode == "2":
sampling = self.sampling_mode_2(frames, self.n_sequence)
else:
raise ValueError
return sampling
def sampling_mode_1(self, chunks):
sampling = []
for i, chunk in enumerate(chunks):
if i == 0 or i == 1:
sampling.append(chunk[-1]) # get the last frame
elif i == (len(chunks) - 1) or i == (len(chunks) - 2):
sampling.append(chunk[0]) # get the first frame
else:
sampling.append(chunk[len(chunk) //
2]) # get the central frame
return sampling
def sampling_mode_2(self, frames, n_sequence):
# create 12 chunks
chunks = list(self.get_chunks(frames, 12))
# remove the first and the last chunk
sub_chunks = chunks[1:-1]
# get a the new list of frames
sub_frame_list = list(itertools.chain.from_iterable(sub_chunks))
# create n_sequence(10) chunks
new_chunks = list(self.get_chunks(sub_frame_list, n_sequence))
sampling = self.sampling_mode_1(new_chunks)
return sampling
def get_chunks(self, l, n):
# divide indexes list in n chunks
k, m = divmod(len(l), n)
return (l[i * k + min(i, m):(i + 1) * k + min(i + 1, m)]
for i in range(n))
def __data_generation(self, list_IDs_temp):
# Generating data with batch size samples
# Initialization
X = np.empty(
(self.batch_size, self.n_sequence, *self.dim, self.n_channels))
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID: path to file
path_file = ID
cap = cv2.VideoCapture(path_file)
# get number of frames
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
index_sampling = self.frame_sampling(length_file) # sampling indxs
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
X[i, j, :, :, :] = new_image
if self.type_gen == "train":
X[i, ] = (self.sampling_augmentation(X[i, ]) / 255.0)
else:
X[i, ] = (X[i, ] / 255.0)
Y[i] = self.labels[ID]
cap.release()
# for debugging
# self.save_frame_sampling(X, i, ID, len(index_sampling))
return X, Y
def sampling_augmentation(self, sequence):
"""
- theta: Float. Rotation angle in degrees.
- tx: Float. Shift in the x direction. - vertical shift (height)
- ty: Float. Shift in the y direction. - horizontal shift (width)
- shear: Float. Shear angle in degrees.
- zx: Float. Zoom in the x direction. - vertical zoom
- zy: Float. Zoom in the y direction. - horizontal zoom
- flip_horizontal: Boolean. Horizontal flip.
- flip_vertical: Boolean. Vertical flip.
- channel_shift_intensity: Float. Channel shift intensity.
- brightness: Float. Brightness shift intensity.
"""
transformations = [
"theta",
"tx",
"ty",
"zx",
"zy",
"flip_horizontal",
"brightness",
]
# random choice of number of transformations
random_transforms = np.random.randint(2, 4) # min 2 - max 3
# random choice of transformations
transforms_idxs = np.random.choice(len(transformations),
random_transforms,
replace=False)
transfor_parameters = {}
for idx in transforms_idxs:
if transformations[idx] == "theta":
transfor_parameters["theta"] = np.random.randint(-5, 5)
elif transformations[idx] == "tx":
transfor_parameters["tx"] = np.random.randint(-10, 10)
elif transformations[idx] == "ty":
transfor_parameters["ty"] = np.random.randint(-15, 15)
elif transformations[idx] == "zx":
transfor_parameters["zx"] = np.random.uniform(0.6, 1.05)
elif transformations[idx] == "zy":
transfor_parameters["zy"] = np.random.uniform(0.6, 1.05)
elif transformations[idx] == "flip_horizontal":
transfor_parameters["flip_horizontal"] = True
elif transformations[idx] == "brightness":
transfor_parameters["brightness"] = np.random.uniform(0.4, 0.6)
len_seq = sequence.shape[0]
for i in range(len_seq):
sequence[i] = self.aug_gen.apply_transform(sequence[i],
transfor_parameters)
return sequence
def save_frame_sampling(self, samp_imgs, i, img_path, n_frames):
# concatenate all frames
train_frame = ()
for n_f in range(n_frames):
train_frame = (
*train_frame,
samp_imgs[i, n_f, ] * 255.0,
)
# get the train of frame in one image
full_img = np.concatenate(train_frame, axis=1)
# info
img_name = (os.path.split(img_path)[1])[:-4]
img_label = os.path.split(os.path.split(img_path)[0])[1]
# save the image
if not os.path.isdir("./sampling_test/"):
os.mkdir("./sampling_test/")
name_file = self.type_gen + "_" + img_label + "_" + img_name
cv2.imwrite("./sampling_test/" + name_file + ".jpg", full_img)
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self,
list_IDs,
labels,
batch_size=32,
dim=(32, 32),
n_channels=1,
n_sequence=4,
shuffle=True,
path_dataset=None,
type_gen='train',
option=None):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_sequence = n_sequence
self.shuffle = shuffle
self.path_dataset = path_dataset
self.type_gen = type_gen
self.option = option
self.aug_gen = ImageDataGenerator()
print("all:", len(self.list_IDs), " batch per epoch",
int(np.floor(len(self.list_IDs) / self.batch_size)))
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
if self.type_gen == 'predict':
return X
else:
return X, y
def get_sampling_frame(self, len_frames):
'''
Sampling n_sequence frame from video file
Input:
len_frames -- number of frames that this video have
Output:
index_sampling -- n_sequence frame indexs from sampling algorithm
'''
# Define maximum sampling rate
random_sample_range = 9
if random_sample_range * self.n_sequence > len_frames:
random_sample_range = len_frames // self.n_sequence
#Randomly choose sample interval and start frame
sample_interval = np.random.randint(3, random_sample_range + 1)
start_i = np.random.randint(
0, len_frames - sample_interval * self.n_sequence + 1)
# Get n_sequence index of frames
index_sampling = []
end_i = sample_interval * self.n_sequence + start_i
for i in range(start_i, end_i, sample_interval):
if len(index_sampling) < self.n_sequence:
index_sampling.append(i)
return index_sampling
def sequence_augment(self, sequence):
name_list = [
'rotate', 'width_shift', 'height_shift', 'brightness',
'flip_horizontal', 'width_zoom', 'height_zoom'
]
dictkey_list = [
'theta', 'ty', 'tx', 'brightness', 'flip_horizontal', 'zy', 'zx'
]
# dictkey_list = ['ty','tx','zy','zx']
random_aug = np.random.randint(2, 5) # random 2-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list),
random_aug,
replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
dict_input['theta'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'ty': # width_shift
dict_input['ty'] = np.random.randint(-60, 60)
elif dictkey_list[i] == 'tx': # height_shift
dict_input['tx'] = np.random.randint(-30, 30)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.15, 1)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = True
elif dictkey_list[i] == 'zy': # width_zoom
dict_input['zy'] = np.random.uniform(0.5, 1.5)
elif dictkey_list[i] == 'zx': # height_zoom
dict_input['zx'] = np.random.uniform(0.5, 1.5)
len_seq = sequence.shape[0]
for i in range(len_seq):
sequence[i] = self.aug_gen.apply_transform(sequence[i], dict_input)
return sequence
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X = np.empty((self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, *dim, n_channels)
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = self.path_dataset + ID
print(path_file)
cap = cv2.VideoCapture(path_file)
print(cap.get(5))
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
new_image = cv2.resize(frame, self.dim)
X[i, j, :, :, :] = new_image
if self.type_gen == 'train':
X[i, ] = self.sequence_augment(X[i, ]) / 255.0
else:
X[i, ] = X[i, ] / 255.0
if self.option == 'RGBdiff':
X[i, ] = calculateRGBdiff(X[i, ])
Y[i] = self.labels[ID]
cap.release()
return X, Y
class DataGenerator2(DataGenerator):
"""Generates data for Keras
Sequence based data generator. Suitable for building data generator for training and prediction.
"""
def __init__(self,
list_IDs,
image_path,
mask_path,
to_fit=True,
batch_size=32,
dim=(512, 512),
dimy=(512, 512),
n_channels=1,
n_classes=10,
shuffle=True,
data_gen_args=None):
super().__init__(list_IDs,
image_path,
mask_path,
to_fit=to_fit,
batch_size=batch_size,
dim=dim,
dimy=dimy,
n_channels=n_channels,
n_classes=n_classes,
shuffle=shuffle)
"""Initialization
:param list_IDs: list of all 'label' ids to use in the generator
:param image_path: path to images location
:param mask_path: path to masks location
:param to_fit: True to return X and y, False to return X only
:param batch_size: batch size at each iteration
:param dim: tuple indicating image dimension
:param n_channels: number of image channels
:param n_classes: number of output masks
:param shuffle: True to shuffle label indexes after every epoch
"""
self.bool = False
if data_gen_args != None:
self.trans = ImageDataGenerator(**data_gen_args)
def __next__(self):
if self.n >= self.max:
self.n = 0
result = self.__getitem__(self.n)
self.n += 1
return result
def _generate_X(self, list_IDs_temp):
"""Generates data containing batch_size images
:param list_IDs_temp: list of label ids to load
:return: batch of images
"""
# Initialization
X = np.empty((self.batch_size, *self.dim, self.n_channels))
self.param = self.trans.get_random_transform(self.dim)
if random.uniform(0, 1) >= 0.5:
self.bool = True
else:
self.bool = False
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
X[i, ] = self._load_dicom_image(self.image_path + '/' + ID)
# X[i,] = self.apply_transform(X[i,],self.get_random_transform((1,512,512)))
if self.bool:
X[i, ] = self.trans.apply_transform(X[i, ], self.param)
# X=np.expand_dims(X, 4)
return X
def _generate_y(self, list_IDs_temp):
"""Generates data containing batch_size masks
:param list_IDs_temp: list of label ids to load
:return: batch if masks
"""
y = np.empty((self.batch_size, *self.dimy, self.n_channels))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
y[i, ] = self._load_grayscale_image_VTK(self.mask_path + '/' +
'label_' + ID[6:15] +
'.png')
if self.bool:
y[i, ] = self.trans.apply_transform(y[i, ], self.param)
return y
class generator3da(generator3d):
def __init__(self,
list_IDs,
image_path,
mask_path,
to_fit=True,
batch_size=32,
patch_size=8,
dim=(512, 512),
dimy=(512, 512),
n_channels=1,
n_classes=10,
shuffle=True,
data_gen_args=None):
super().__init__(list_IDs,
image_path,
mask_path,
to_fit=to_fit,
batch_size=batch_size,
patch_size=patch_size,
dim=dim,
dimy=dimy,
n_channels=n_channels,
n_classes=n_classes,
shuffle=shuffle)
"""Initialization
:param list_IDs: list of all 'label' ids to use in the generator
:param image_path: path to images location
:param mask_path: path to masks location
:param to_fit: True to return X and y, False to return X only
:param batch_size: batch size at each iteration
:param dim: tuple indicating image dimension of the input
:param dimy: tuple indicating image dimension of the output
:param n_channels: number of image channels
:param n_classes: number of output masks
:param shuffle: True to shuffle label indexes after every epoch
"""
if data_gen_args != None:
self.trans = ImageDataGenerator(**data_gen_args)
def _generate_y(self, list_IDs_temp):
"""Generates data containing batch_size images
:param list_IDs_temp: list of label ids to load
:return: batch of images
"""
Y = np.zeros(
(self.batch_size, *self.dimy, self.patch_size, self.n_channels))
# Generate data
for patch in list_IDs_temp:
for i, ID in enumerate(patch[1]):
path = self.mask_path + '/' + patch[0] + '/' + 'label_' + ID[
6:15] + '.png'
img = self._load_grayscale_image_VTK(path)[:, :, 0]
Y[0, :, :, i, 0] = img
if self.bool:
Y[0, :, :, i, :] = self.trans.apply_transform(
Y[0, :, :, i, :], self.param)
return Y
def _generate_X(self, list_IDs_temp):
"""Generates data containing batch_size images
:param list_IDs_temp: list of label ids to load
:return: batch of images
"""
X = np.zeros(
(self.batch_size, *self.dim, self.patch_size, self.n_channels))
self.param = self.trans.get_random_transform(self.dim)
if random.uniform(0, 1) >= 0.5:
self.bool = True
else:
self.bool = False
# Generate data
for patch in list_IDs_temp:
for i, ID in enumerate(patch[1]):
path = self.image_path + '/' + patch[0] + '/' + ID
img = self._load_grayscale_image_VTK(path)[:, :, 0]
X[0, :, :, i, 0] = img
if self.bool:
X[0, :, :, i, :] = self.trans.apply_transform(
X[0, :, :, i, :], self.param)
return X
class OmniglotLoader():
'''
A class which handles operations for loading and transforming images from Omniglot dataset.
'''
def __init__(self,
path='./Omniglot',
rotation_range=[-10, 10],
shear_range=[-15, 15],
scale_range=[0.8, 1.2],
shift_range=[-2, 2],
batch_size=20,
use_transformations=True):
'''
Basic constructor which creates the class and sets up all the parameters necessary for running the training
Params:
- path = path to root folder of omniglot dataset
- rotation_range = range of rotations to be applied to the images
- shear_range = range of degrees to be applied for shearing
- scale_range = range of factor for scaling by x and y
- shift_range = range of factor for shifting by x and y
- batch_size = amount of pairs to be sent in one batch
- use_transformations = should OmniglotLoader apply random transformations to images
'''
self.path = path
self.rotation_range = rotation_range
self.shear_range = shear_range
self.scale_range = scale_range
self.shift_range = shift_range
self.batch_size = batch_size
# Directories for training and evaluation dataset
self.background_dir = 'images_background'
self.evaluation_dir = 'images_evaluation'
self.use_transformations = use_transformations
# Reading alphabets and creating dictinaries
self.training_alphabets, self.evaluation_alphabets = self._create_alphabets(
)
self.training_keys, self.testing_keys = self._split_train_sets()
# Creating a series of lists and indexes that will be used during sending batches
# Because it is important to go through entire dataset for each epoch,
# it is necessary to keep track of where we are for each batch.
self._current_alphabet_index = 0
self._training_alphabet_list = list(self.training_alphabets.keys())
self._symbols_list = list(self.training_alphabets[
self._training_alphabet_list[self._current_alphabet_index]].keys())
self._current_symbol_index = 0
# Flag to indicate if the current epoch is done
self._epoch_done = False
self._training_alphabet_num = len(self.training_alphabets.keys())
self._evaluation_alphabet_list = list(self.evaluation_alphabets.keys())
self._evaluation_alphabet_num = len(self._evaluation_alphabet_list)
self._evaluation_done = False
self._tp_batches_done = False
self._tn_batches_done = False
self._training = False
self._testing = False
self.img_transformer = ImageDataGenerator()
####################################################################
# Getters and setters #
####################################################################
def set_training_evaluation_symbols(self, training):
if training:
self._symbols_list = list(
self.training_alphabets[self._training_alphabet_list[
self._current_alphabet_index]].keys())
else:
self._symbols_list = list(
self.evaluation_alphabets[self._evaluation_alphabet_list[
self._current_alphabet_index]].keys())
def get_alphabet_index(self, alphabet, training):
if training:
return self._training_alphabet_list.index(alphabet)
else:
return self._evaluation_alphabet_list.index(alphabet)
####################################################################
# Various public functions #
####################################################################
def create_pairs(self, directory, alphabets, keys):
image_alphabets = {}
for alphabet in keys:
symbol_dict = {}
for symbol in alphabets[alphabet]:
symbol_paths = [
os.path.join(self.path, directory, alphabet, symbol, img)
for img in os.listdir(
os.path.join(self.path, directory, alphabet, symbol))
]
symbol_dict[symbol] = [
self._get_image(img_path) for img_path in symbol_paths
]
image_alphabets[alphabet] = symbol_dict
pairs = []
labels = []
for alphabet in image_alphabets:
for symbol in image_alphabets[alphabet].keys():
for img in image_alphabets[alphabet][symbol]:
same_imgs = random.sample(range(0, 20), 10)
test_case = img
for i in same_imgs:
image = image_alphabets[alphabet][symbol][i]
if self.use_transformations:
image = self._transform_image(image)
pairs += [[test_case, image]]
random_img = self._get_random_image(
alphabet, image_alphabets, self.background_dir)
pairs += [[test_case, random_img]]
labels += [1, 0]
return np.array(pairs), np.array(labels)
####################################################################
# Functions for getting various batches #
####################################################################
def get_training_batch(self):
'''
Returns one batch from training dataset. Alternate between positive (1) and negative samples (0).
'''
directory = self.background_dir
pairs = []
labels = []
images = random.sample(range(0, 20), int((self.batch_size / 2) + 1))
current_alphabet = self._training_alphabet_list[
self._current_alphabet_index]
current_symbol = self._symbols_list[self._current_symbol_index]
current_image = self._get_image(
os.path.join(
self.path, directory, current_alphabet, current_symbol,
self.training_alphabets[current_alphabet][current_symbol][0]))
for img in images[1:]:
second_image = self._get_image(
os.path.join(
self.path, directory, current_alphabet, current_symbol,
self.training_alphabets[current_alphabet][current_symbol]
[img]))
pairs += [[current_image, second_image]]
random_img = self._get_random_image(current_alphabet,
self.training_alphabets,
self.background_dir)
pairs += [[current_image, random_img]]
labels += [1, 0]
if self.use_transformations:
pairs += [[current_image, self._transform_image(second_image)]]
pairs += [[current_image, self._transform_image(random_img)]]
labels += [1, 0]
self._change_symbol()
return np.array(pairs), np.array(labels)
def get_random_batch(self):
'''
Gets one random batch with images and their labels. Alternate between positive and negative labels.
Returns:
- pairs = pairs of images
- labels = labels for pairs
'''
pairs = []
labels = []
images = random.sample(range(0, 20), int((self.batch_size / 2) + 1))
# Get one random image of a current symbol, whether it is training or evaluation symbol.
if self._testing:
random_alphabet_idx = random.randint(
0,
len(self._evaluation_alphabet_list) - 1)
random_alphabet = self._evaluation_alphabet_list[
random_alphabet_idx]
symbols_list = list(
self.evaluation_alphabets[random_alphabet].keys())
random_symbol_idx = random.randint(0, len(symbols_list) - 1)
random_symbol = symbols_list[random_symbol_idx]
current_image = self._get_image(
os.path.join(
self.path, self.evaluation_dir, random_alphabet,
random_symbol, self.evaluation_alphabets[random_alphabet]
[random_symbol][0]))
else:
random_alphabet_idx = random.randint(
0,
len(self._training_alphabet_list) - 1)
random_alphabet = self._training_alphabet_list[random_alphabet_idx]
symbols_list = list(
self.training_alphabets[random_alphabet].keys())
random_symbol_idx = random.randint(0, len(symbols_list) - 1)
random_symbol = symbols_list[random_symbol_idx]
current_image = self._get_image(
os.path.join(
self.path, self.background_dir, random_alphabet,
random_symbol, self.training_alphabets[random_alphabet]
[random_symbol][0]))
for img in images[1:]:
# Get one of the other images and create positive pair.
if self._testing:
second_image = self._get_image(
os.path.join(
self.path, self.evaluation_dir, random_alphabet,
random_symbol,
self.evaluation_alphabets[random_alphabet]
[random_symbol][img]))
else:
second_image = self._get_image(
os.path.join(
self.path, self.background_dir, random_alphabet,
random_symbol, self.training_alphabets[random_alphabet]
[random_symbol][img]))
pairs += [[current_image, second_image]]
# Get one random image to create negative pair.
if self._testing:
random_img = self._get_random_image(random_alphabet,
self.evaluation_alphabets,
self.evaluation_dir)
else:
random_img = self._get_random_image(random_alphabet,
self.training_alphabets,
self.background_dir)
pairs += [[current_image, random_img]]
labels += [1, 0]
if self.use_transformations:
pairs += [[current_image, self._transform_image(second_image)]]
pairs += [[current_image, self._transform_image(random_img)]]
labels += [1, 0]
return np.array(pairs), np.array(labels)
def get_positive_batch(self):
'''
Gets a batch of positive pairs from dataset.
Arguments:
- training = flag indicating should the training or evaluation dataset be used.
Returns:
- pairs = pairs of images
- labels = labels for pairs
'''
pairs = []
labels = []
random_pairs = [(random.randint(0, self.batch_size - 1),
random.randint(0, self.batch_size - 1))
for k in range(self.batch_size)]
if self._training:
current_alphabet = self._training_alphabet_list[
self._current_alphabet_index]
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.background_dir
img_names = self.training_alphabets[current_alphabet][
current_symbol]
else:
current_alphabet = self._evaluation_alphabet_list[
self._current_alphabet_index]
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.evaluation_dir
img_names = self.evaluation_alphabets[current_alphabet][
current_symbol]
for pair in random_pairs:
left_image = self._get_image(
os.path.join(self.path, directory, current_alphabet,
current_symbol, img_names[pair[0]]))
right_image = self._get_image(
os.path.join(self.path, directory, current_alphabet,
current_symbol, img_names[pair[1]]))
if self.use_transformations:
left_image = self._transform_image(left_image)
right_image = self._transform_image(right_image)
pairs += [[left_image, right_image]]
labels += [1]
if self.use_transformations:
pairs += [[
self._transform_image(left_image),
self._transform_image(right_image)
]]
labels += [1]
self._change_symbol(training)
return np.array(pairs), np.array(labels)
def get_negative_batch(self):
'''
Gets a batch of negative pairs from dataset.
Arguments:
- training = flag indicating should the training or evaluation dataset be used.
Returns:
- pairs = pairs of images
- labels = labels for pairs
'''
pairs = []
labels = []
image_idx = random.randint(0, self.batch_size - 1)
if self._training:
current_alphabet = self._training_alphabet_list[
self._current_alphabet_index]
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.background_dir
img_names = self.training_alphabets[current_alphabet][
current_symbol]
alphabet_dict = self.training_alphabets
else:
current_alphabet = self._evaluation_alphabet_list[
self._current_alphabet_index]
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.evaluation_dir
img_names = self.evaluation_alphabets[current_alphabet][
current_symbol]
alphabet_dict = self.evaluation_alphabets
for _ in range(0, self.batch_size - 1):
left_image = self._get_image(
os.path.join(self.path, directory, current_alphabet,
current_symbol, img_names[image_idx]))
right_image = self._get_random_image(current_alphabet,
alphabet_dict, directory)
pairs += [[left_image, right_image]]
labels += [0]
if self.use_transformations:
pairs += [[
self._transform_image(left_image),
self._transform_image(right_image)
]]
labels += [0]
self._change_symbol(training)
return np.array(pairs), np.array(labels)
def get_test_batch(self):
'''
Gets one batch from evaluation set.
Returns:
- pairs = pairs of images
- labels = labels for pairs
'''
directory = self.evaluation_dir
pairs = []
labels = []
images = random.sample(range(0, 20), 11)
current_alphabet = self._evaluation_alphabet_list[
self._current_alphabet_index]
current_symbol = self._symbols_list[self._current_symbol_index]
current_image = self._get_image(
os.path.join(
self.path, directory, current_alphabet, current_symbol,
self.evaluation_alphabets[current_alphabet][current_symbol]
[0]))
for img in images[1:]:
second_image = self._get_image(
os.path.join(
self.path, directory, current_alphabet, current_symbol,
self.evaluation_alphabets[current_alphabet][current_symbol]
[img]))
pairs += [[current_image, second_image]]
random_img = self._get_random_image(current_alphabet,
self.evaluation_alphabets,
self.evaluation_dir)
pairs += [[current_image, random_img]]
labels += [1, 0]
if self.use_transformations:
pairs += [[current_image, self._transform_image(second_image)]]
pairs += [[current_image, self._transform_image(random_img)]]
labels += [1, 0]
self._change_symbol(False)
if self._epoch_done:
self._evaluation_done = True
return np.array(pairs), np.array(labels)
def get_tp_batch(self, alphabet_name):
'''
Helper function to get true positive pairs from alphabet of interest in.
Arguments:
- training = flag indicating should the training or evaluation dataset be used.
Returns:
- pairs = pairs of images
- labels = labels for pairs
'''
pairs = []
labels = []
random_pairs = [(random.randint(0, self.batch_size - 1),
random.randint(0, self.batch_size - 1))
for k in range(self.batch_size)]
if self._training:
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.background_dir
img_names = self.training_alphabets[alphabet_name][current_symbol]
else:
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.evaluation_dir
img_names = self.evaluation_alphabets[alphabet_name][
current_symbol]
for pair in random_pairs:
left_image = self._get_image(
os.path.join(self.path, directory, alphabet_name,
current_symbol, img_names[pair[0]]))
right_image = self._get_image(
os.path.join(self.path, directory, alphabet_name,
current_symbol, img_names[pair[1]]))
pairs += [[left_image, right_image]]
labels += [1]
if self.use_transformations:
pairs += [[
self._transform_image(left_image),
self._transform_image(right_image)
]]
labels += [1]
self._current_symbol_index += 1
if self._training:
if self._current_symbol_index == len(
self.training_alphabets[alphabet_name]):
self._tp_batches_done = True
else:
if self._current_symbol_index == len(
self.evaluation_alphabets[alphabet_name]):
self._tp_batches_done = True
return np.array(pairs), np.array(labels)
def get_tn_batch(self, alphabet_name):
'''
Gets a batch of negative pairs from dataset.
Arguments:
- training = flag indicating should the training or evaluation dataset be used.
Returns:
- pairs = pairs of images
- labels = labels for pairs
'''
pairs = []
labels = []
image_idx = random.randint(0, self.batch_size - 1)
if self._training:
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.background_dir
img_names = self.training_alphabets[alphabet_name][current_symbol]
alphabet_dict = self.training_alphabets
else:
current_symbol = self._symbols_list[self._current_symbol_index]
directory = self.evaluation_dir
img_names = self.evaluation_alphabets[alphabet_name][
current_symbol]
alphabet_dict = self.evaluation_alphabets
for _ in range(0, self.batch_size - 1):
left_image = self._get_image(
os.path.join(self.path, directory, alphabet_name,
current_symbol, img_names[image_idx]))
right_image = self._get_random_image(alphabet_name, alphabet_dict,
directory)
pairs += [[left_image, right_image]]
labels += [0]
if self.use_transformations:
pairs += [[
self._transform_image(left_image),
self._transform_image(right_image)
]]
labels += [0]
self._current_symbol_index += 1
if self._training:
if self._current_symbol_index == len(
self.training_alphabets[alphabet_name]):
self._tn_batches_done = True
else:
if self._current_symbol_index == len(
self.evaluation_alphabets[alphabet_name]):
self._tn_batches_done = True
return np.array(pairs), np.array(labels)
####################################################################
# Various private functions #
####################################################################
def _transform_image(self, img):
'''
Function that performs random affine transformations for given image.
Transformation will occur with probability of 50%.
Arguments:
- img = image to be transformed
Returns:
- transformed = transformed image
'''
transformations = {}
if np.random.uniform(low=0, high=1) < 0.5:
theta = np.random.uniform(low=self.rotation_range[0],
high=self.rotation_range[1])
transformations['theta'] = theta
if np.random.uniform(low=0, high=1) < 0.5:
tx = np.random.uniform(low=self.shift_range[0],
high=self.shift_range[1])
transformations['tx'] = tx
if np.random.uniform(low=0, high=1) < 0.5:
ty = np.random.uniform(low=self.shift_range[0],
high=self.shift_range[1])
transformations['ty'] = ty
if np.random.uniform(low=0, high=1) < 0.5:
zx = np.random.uniform(low=self.scale_range[0],
high=self.scale_range[1])
transformations['zx'] = zx
if np.random.uniform(low=0, high=1) < 0.5:
zy = np.random.uniform(low=self.scale_range[0],
high=self.scale_range[1])
transformations['zy'] = zy
if np.random.uniform(low=0, high=1) < 0.5:
shear = np.random.uniform(low=self.shear_range[0],
high=self.shear_range[1])
transformations['shear'] = shear
transformed = self.img_transformer.apply_transform(
img, transformations)
return transformed
def _get_image(self, path):
'''
Gets image from given path. Since Omniglot images are black with white background,
image will be inverted to be easier to use with affine transformations
Arguments:
- path = path to image
Returns:
- inverted = transformed image
'''
img = plt.imread(path)
img = img.reshape(105, 105, 1)
return img
def _change_symbol(self):
'''
Private function to be used for selecting next symbol after all the data for batch has been prepared.
Arguments:
- training = flag indicating should the training or evaluation dataset be used.
'''
self._current_symbol_index += 1
if self._training:
current_alphabet = self._training_alphabet_list[
self._current_alphabet_index]
if self._current_symbol_index == len(
self.training_alphabets[current_alphabet]):
# If we reached last symbol, it is neccessary to reset the counter and change the alphabet.
self._current_symbol_index = 0
self._current_alphabet_index += 1
print(
str(
round((self._current_alphabet_index /
self._training_alphabet_num) * 100.00, 2)) +
' %% of alphabets done')
if self._current_alphabet_index == len(
self.training_alphabets):
# If we reached last alphabet, it is neccessary to reset the counter and start a new epoch.
self._current_alphabet_index = 0
self._epoch_done = True
self._symbols_list = list(
self.training_alphabets[self._training_alphabet_list[
self._current_alphabet_index]].keys())
else:
current_alphabet = self._evaluation_alphabet_list[
self._current_alphabet_index]
if self._current_symbol_index == len(
self.evaluation_alphabets[current_alphabet]):
# If we reached last symbol, it is neccessary to reset the counter and change the alphabet.
self._current_symbol_index = 0
self._current_alphabet_index += 1
print(
str(
round((self._current_alphabet_index /
self._evaluation_alphabet_num) * 100.00, 2)) +
' %% of alphabets done')
if self._current_alphabet_index == len(
self.evaluation_alphabets):
# If we reached last alphabet, it is neccessary to reset the counter and start a new epoch.
self._current_alphabet_index = 0
self._epoch_done = True
self._symbols_list = list(
self.evaluation_alphabets[self._evaluation_alphabet_list[
self._current_alphabet_index]].keys())
def _get_random_image(self, current_alphabet, alphabet_dict, directory):
"""
Gets a random image of a random symbol from a random alphabet.
Current alphabet is excluded.
Params:
- current_alphabet = name of current alphabet
- alphabet_dict = dictionary of alphabets
- directory = string that points to the evaluation or training directory
"""
keys = set(alphabet_dict.keys())
keys.remove(current_alphabet)
alphabet_sample = keys
random_alphabet = random.choice(list(alphabet_sample))
random_symbol = random.choice(list(alphabet_dict[random_alphabet]))
random_idx = random.randint(0, 19)
random_img = self._get_image(
os.path.join(
self.path, directory, random_alphabet, random_symbol,
alphabet_dict[random_alphabet][random_symbol][random_idx]))
return random_img
def _create_alphabets(self):
'''
Arrange all the images of symbols into lists and dictionaries.
The function reads all the images sorted by folders.
The folder structure for Omniglot dataset should look like this
Omniglot
|-images_background
|-Arcadian
|-character01
|-img01.png
|-img02.png
...
|-images_evaluation
'''
training_path = os.path.join(self.path, self.background_dir)
training_alphabets = {}
for alphabet in os.listdir(training_path):
print('Processing: ' + alphabet)
characters = {}
for char in os.listdir(os.path.join(training_path, alphabet)):
characters[char] = [
img for img in os.listdir(
os.path.join(training_path, alphabet, char))
]
training_alphabets[alphabet] = characters
evaluation_path = os.path.join(self.path, self.evaluation_dir)
evaluation_alphabets = {}
for alphabet in os.listdir(evaluation_path):
print('Processing: ' + alphabet)
characters = {}
for char in os.listdir(os.path.join(evaluation_path, alphabet)):
characters[char] = [
img for img in os.listdir(
os.path.join(evaluation_path, alphabet, char))
]
evaluation_alphabets[alphabet] = characters
return (training_alphabets, evaluation_alphabets)
def _split_train_sets(self):
"""
Perform training and testing split at 80% - 20%"
"""
alphabet_indices = list(range(len(self.training_alphabets)))
training_indices = random.sample(
range(0,
len(self.training_alphabets) - 1),
k=int(0.8 * len(self.training_alphabets)))
testing_indices = set(alphabet_indices) - set(training_indices)
training_keys = [
list(self.training_alphabets.keys())[i] for i in training_indices
]
testing_keys = [
list(self.training_alphabets.keys())[i] for i in testing_indices
]
return (training_keys, testing_keys)
class HeadPoseDataGenerator(Sequence):
'''
This class implements a basic Keras data generator overriding methods from its parent class Sequence. The purpose
of this data generator is to deliver in each batch a set of images from the subset used to initalize this generator
containing an equal number of members per class.
'''
def __init__(self, pose_dataframe, img_array, batch_size,
normalize=False, input_norm=None, tilt_norm=None, pan_norm=None,
augment=False, shift_range=None, zoom_range=None, brightness_range=None,
img_rescale=1, out_rescale=1):
'''
Initializes the data generator with the data from a given subset from the original dataset, and the values to
use when doing data augmentation.
Arguments:
pose_dataframe: Dataframe containing a list of each picture in the given subset and its pose values.
img_array: Numpy array containing the images from the given subset.
batch_size: Number of pictures per batch.
normalize: If the data shall be normalized or not.
input_norm: Tuple containing mean and std values for normalizing pictures in the dataset.
tilt_norm: Tuple containing mean and std values for normalizing tilt values in the dataset.
pan_norm: Tuple containing mean and std values for normalizing pan values in the dataset.
augment: If data augmentation shall be applied or not.
shift_range: Value (between 0 and 1) indicating the portion of the length of the side of each picture that
can be used to shift the picture (in both axes).
zoom_range: Tuple containing the minimum and maximum values used to apply zoom to each picture.
brightness_range: Tuple containing the minimum and maximum values used to apply a brightness transformation
to each picture.
img_rescale: Each pixel from every picture in the subset will be multiplied by this value.
out_rescale: Tilt and pan values for every picture in the subset will be multiplied by this value.
'''
# Create empty arrays for pictures and labels from the subset.
self.pics = []
self.labels = []
# Initialize batch size.
self.batch_size = batch_size
# Initialize normalization parameters.
self.normalize = normalize
self.input_norm = input_norm
'''
Initialize the parameter controlling if data augmentation shall be applied or not, and data augmentation
parameters.
'''
self.augment = augment
if self.augment == True:
self.generator = ImageDataGenerator(width_shift_range=shift_range, height_shift_range=shift_range,
brightness_range=brightness_range, zoom_range=zoom_range)
# Initialize scaling parameters.
self.img_rescale = img_rescale
self.out_rescale = out_rescale
'''
Initialize the iterator used to control the position of the next picture from every class that will be included
in a batch.
'''
self.common_iterator = 0
# Sort dataframe by class.
df = pose_dataframe.sort_values('class')
# Initialize the number of pictures in the dataset.
self.total_size = len(df.index)
# Load images and pose values into the previously created arrays.
prev_class = -1
class_index = -1
# For each image in the (ordered) dataset:
for index, row in df.iterrows():
'''
If the class for the current picture is different from the last class recorded, append an empty list for the
new class.
'''
if row['class'] != prev_class:
prev_class = row['class']
self.pics.append([])
self.labels.append([])
class_index = class_index + 1
# Append picture to corresponding class array.
self.pics[class_index].append(np.squeeze(img_array[index]))
# Append labels to corresponding class array (normalized and rescaled).
self.labels[class_index].append([((row['tilt'] * out_rescale) - tilt_norm[0]) / tilt_norm[1] , ((row['pan'] * out_rescale) - pan_norm[0]) / pan_norm[1]])
# Assert batch size is a multiple of the number of classes.
assert(batch_size % len(self.pics) == 0)
def __data_generation(self):
'''
Outputs a batch of pictures.
Returns:
X: Pictures in the batch.
y: Labels for each picture in the batch.
'''
# Create empty lists for pictures and labels.
X = []
y = []
# For each picture-per-class:
for i in range(int(self.batch_size / len(self.pics))):
# For each class:
for j in range(len(self.pics)):
# Select the next picture in the class list (start from beginning after the last picture).
pic = self.pics[j][int(self.common_iterator % len(self.pics[j]))]
pic = np.expand_dims(pic, axis=2)
# Apply data augmentation.
if self.augment == True:
transformation = self.generator.get_random_transform(pic.shape)
transformation['zx'] = transformation['zy']
pic = self.generator.apply_transform(pic, transformation)
# Rescale each pixel value in image.
pic = pic * self.img_rescale
# Normalize image.
if self.normalize == True:
pic = (pic - self.input_norm[0]) / self.input_norm[1]
# Add image and labels to the batch.
X.append(pic)
y.append(self.labels[j][int(self.common_iterator % len(self.labels[j]))])
# Update iterator.
self.common_iterator = self.common_iterator + 1
# Transform lists into Numpy arrays.
X = np.array(X)
y = np.array(y)
# Return images and labels.
return X, y
def __len__(self):
'''
Outputs the length (number of batches) that the data generator can provide.
Returns:
l: The length of the data generator.
'''
'''
Calculate the length of the data generator as the relation between the total number of images and the size of
each batch; in order to function properly with uneven class lengths the number is rounded to the smaller integer
bigger or equal to the obtained result.
'''
l = ceil(self.total_size / self.batch_size)
return l
def __getitem__(self, index):
'''
Outputs a new batch given the batch index.
Returns:
X: Pictures in the batch.
y: Labels for each picture in the batch.
'''
# Set the class iterator in the correct position for obtaining the requested batch.
self.common_iterator = index * int(self.batch_size / len(self.pics))
# Generate the batch.
X, y = self.__data_generation()
# Return images and labels for the requested batch.
return X, y
def reset(self):
'''
Resets the class iterator to its initial position.
'''
self.common_iterator = 0
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.01,
zoom_range=[0.9, 1.25],
horizontal_flip=True,
vertical_flip=False,
fill_mode='reflect',
data_format='channels_last',
brightness_range=[0.5, 1.5])
cv2.imwrite(
'/Users/denisrangulov/Google Drive/EmotionRecognition/figures/origin.png',
image)
datagen = ImageDataGenerator()
aug_brightness = datagen.apply_transform(
x=image, transform_parameters={'brightness': 1.5})
cv2.imwrite(
'/Users/denisrangulov/Google Drive/EmotionRecognition/figures/aug_brightness.png',
aug_brightness)
aug_rotation = datagen.apply_transform(x=image,
transform_parameters={'theta': 15})
cv2.imwrite(
'/Users/denisrangulov/Google Drive/EmotionRecognition/figures/aug_rotation.png',
aug_rotation)
aug_shift = datagen.apply_transform(x=image,
transform_parameters={
'tx': 5,
'ty': 5
})
class DataGeneratorBKB(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self,
list_IDs,
labels,
batch_size=32,
dim=(32, 32),
n_channels=1,
n_sequence=4,
shuffle=True,
path_dataset=None,
select_joint=[],
type_gen='train',
type_model='rgb',
option=None):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_sequence = n_sequence
self.shuffle = shuffle
self.path_dataset = path_dataset
self.select_joint = select_joint
self.n_joint = len(select_joint)
self.option = option
self.type_gen = type_gen
self.type_model = type_model
print("all:", len(self.list_IDs), " batch per epoch",
int(np.floor(len(self.list_IDs) / self.batch_size)))
# execution_path = os.getcwd()
# self.detector = ObjectDetection()
# self.detector.setModelTypeAsYOLOv3()
# self.detector.setModelPath( os.path.join(execution_path , "pretrain/yolo.h5"))
# self.detector.loadModel(detection_speed="fast")#detection_speed="fast"
# self.execution_path = execution_path
# self.detector = detector
# sometimes = lambda aug: va.Sometimes(0.5, aug) # Used to apply augmentor with 50% probability
# self.seq = va.SomeOf([ #va.Sequential([
# # va.RandomCrop(size=(300, 300)), # randomly crop video with a size of (240 x 180)
# va.RandomRotate(degrees=10), # randomly rotates the video with a degree randomly choosen from [-10, 10]
# va.RandomTranslate(x=60,y=30),
# # sometimes(va.Add(value=-100)),
# # sometimes(va.Pepper(ratio=40)),
# sometimes(va.Add(value=-60)),
# sometimes(va.HorizontalFlip()) # horizontally flip the video with 50% probability
# ], 2)
self.aug_gen = ImageDataGenerator()
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
if self.type_gen == 'predict':
return X
else:
return X, y
def get_sampling_frame(self, len_frames, path_video):
'''
Sampling n_sequence frame from video file
Input:
len_frames -- number of frames that this video have
Output:
index_sampling -- n_sequence frame indexs from sampling algorithm
'''
# Define maximum sampling rate
# sample_interval = len_frames//self.n_sequence
# start_i = 0 #np.random.randint(0, len_frames - sample_interval * self.n_sequence + 1)
if True: #self.type_gen =='train':
random_sample_range = 10
if random_sample_range * self.n_sequence > len_frames:
random_sample_range = len_frames // self.n_sequence
if random_sample_range <= 0:
print('test:', random_sample_range, len_frames, path_video)
# Randomly choose sample interval and start frame
if random_sample_range < 3:
sample_interval = np.random.randint(1, random_sample_range + 1)
else:
sample_interval = np.random.randint(3, random_sample_range + 1)
# sample_interval = np.random.randint(1, random_sample_range + 1)
# temp = len_frames - sample_interval * self.n_sequence + 1
# if temp <= 0:
# print(temp, len_frames)
start_i = np.random.randint(
0, len_frames - sample_interval * self.n_sequence + 1)
# Get n_sequence index of frames
index_sampling = []
end_i = sample_interval * self.n_sequence + start_i
for i in range(start_i, end_i, sample_interval):
if len(index_sampling) < self.n_sequence:
index_sampling.append(i)
return index_sampling
def get_crop_img(self, frame):
# detect_image, detections, extract_picture = self.detector.detectObjectsFromImage(input_type="array", input_image=frame, output_type='array',
# minimum_percentage_probability=10, extract_detected_objects=True )
print('#################', self.execution_path)
detections = self.detector.detectObjectsFromImage(
input_image=os.path.join(self.execution_path, "room.jpg"),
output_image_path=os.path.join(self.execution_path,
"image2new.jpg"),
minimum_percentage_probability=30)
max_prob = 0
max_idx = 0
for i, eachObject in enumerate(detections):
if eachObject["name"] == 'person' and eachObject[
"percentage_probability"] > max_prob:
max_prob = eachObject["percentage_probability"]
max_idx = i
if max_idx > len(detections):
# if no detection, use black array
crop_img = np.zeros((*self.dim, self.n_channels))
else:
crop_img = extract_picture[max_idx]
return crop_img
# def calculateRGBdiff(self, sequence_img):
# 'keep first frame as rgb data, other is use RGBdiff for temporal data'
# length = len(sequence_img)
# new_sequence = np.zeros((length,self.dim[0],self.dim[1],self.n_channels))
# # find RGBdiff frame 1 to last frame
# for i in range(length-1,3,-1): # count down
# new_sequence[i] = cv2.subtract(sequence_img[i],sequence_img[i-1])
# new_sequence[:4] = sequence_img[:4] # first frame as rgb data
# return new_sequence
def sequence_augment(self, sequence):
name_list = [
'rotate', 'width_shift', 'height_shift', 'brightness',
'flip_horizontal', 'width_zoom', 'height_zoom'
]
dictkey_list = [
'theta', 'ty', 'tx', 'brightness', 'flip_horizontal', 'zy', 'zx'
]
# dictkey_list = ['ty','tx','zy','zx']
random_aug = np.random.randint(2, 5) # random 0-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list),
random_aug,
replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
# dict_input['theta'] = np.random.randint(-10, 10)
dict_input['theta'] = np.random.randint(-5, 5)
elif dictkey_list[i] == 'ty': # width_shift
# dict_input['ty'] = np.random.randint(-60, 60)
dict_input['ty'] = np.random.randint(-20, 20)
elif dictkey_list[i] == 'tx': # height_shift
# dict_input['tx'] = np.random.randint(-30, 30)
dict_input['tx'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.15, 1)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = True
elif dictkey_list[i] == 'zy': # width_zoom
# dict_input['zy'] = np.random.uniform(0.5,1.5)
dict_input['zy'] = np.random.uniform(0.9, 1.3)
elif dictkey_list[i] == 'zx': # height_zoom
# dict_input['zx'] = np.random.uniform(0.5,1.5)
dict_input['zx'] = np.random.uniform(0.9, 1.3)
sh = sequence.shape
new_sequence = np.zeros((sh[0], sh[1], sh[2], sh[3]))
for i in range(sh[0]):
new_sequence[i] = self.aug_gen.apply_transform(
sequence[i], dict_input)
return new_sequence
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
X1 = np.empty(
(self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # X : (n_samples, timestep, *dim, n_channels)
X2 = np.empty((self.batch_size, self.n_sequence, self.n_joint * 3))
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file (2 batch)
path_video = self.path_dataset + ID + '.mp4'
# print(path_video)
path_skeleton = self.path_dataset + ID + '.npy'
# print(path_video)
if self.type_model == '2stream' or self.type_model == 'rgb':
cap = cv2.VideoCapture(path_video)
length_file = int(
cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have ,-1 because some bug
if self.type_model == '2stream' or self.type_model == 'skeleton':
skeleton_data = np.load(path_skeleton)
length_file = skeleton_data.shape[0]
index_sampling = self.get_sampling_frame(
length_file, path_video) # get sampling index
# print(index_sampling)
if self.type_model == '2stream' or self.type_model == 'rgb':
# Get RGB sequence
for j, n_pic in enumerate(index_sampling):
cap.set(cv2.CAP_PROP_POS_FRAMES,
n_pic) # jump to that index
ret, frame = cap.read()
# frame = self.get_crop_img(frame)
new_image = cv2.resize(frame, self.dim)
# new_image = frame
# new_image = new_image/255.0
X1[i, j, :, :, :] = new_image
if self.type_gen == 'train':
X1[i, ] = self.sequence_augment(X1[i, ]) / 255.0 * 2 - 1
else:
X1[i, ] = X1[i, ] / 255.0 * 2 - 1
# cv2.imshow('imgae',X1[i,0])
# cv2.waitKey(2000)
if self.option == 'RGBdiff':
# print("dddddddddddd")
X1[i, ] = calculateRGBdiff(X1[i, ], 0)
if self.type_model == '2stream' or self.type_model == 'skeleton':
# Get skeleton sequence
skeleton_data = skeleton_data[index_sampling]
skeleton_data = skeleton_data[:, :, self.select_joint]
skeleton_data = skeleton_data.reshape(self.n_sequence,
self.n_joint * 3)
X2[i] = skeleton_data
# Get label
Y[i] = self.labels[ID]
if self.type_model == '2stream' or self.type_model == 'rgb':
cap.release()
# for i_frame in range(self.n_sequence):
# cv2.imshow('Frame', X1[i,i_frame])
# cv2.waitKey(1000)
if self.type_model == 'rgb':
X = X1
elif self.type_model == 'skeleton':
X = X2
elif self.type_model == '2stream':
X = [X1, X2]
return X, Y
class UnetDataGenerator(Sequence):
"""
Inspired on this thread: https://github.com/keras-team/keras/issues/12120
"""
def __init__(self,
features,
targets,
image_size,
label_size,
classes,
batch_size,
repeats=1,
shuffle=True,
validation=False):
self._features = features
self._targets = targets
self._image_size = image_size
self._label_size = label_size
self._classes = classes
self._batch_size = batch_size
self._shuffle = shuffle
self._repeats = repeats
self._verification = validation
self._ids = list(self._list_files())
self._indexes = np.arange(len(self._ids))
if self._shuffle:
np.random.shuffle(self._indexes)
self._indexes = np.repeat(self._indexes, self._repeats)
self._augmenter = ImageDataGenerator(rotation_range=30,
zoom_range=0.2,
width_shift_range=0.01,
height_shift_range=0.01,
horizontal_flip=True,
vertical_flip=True,
samplewise_std_normalization=True)
def _list_files(self):
feature_files = os.listdir(self._features)
target_files = os.listdir(self._targets)
return zip(feature_files, target_files)
def _encode_one_hot(self, label):
x = np.zeros(
(self._label_size[0] * self._label_size[1], self._classes))
for c in range(1, self._classes + 1):
a = label == c
x[(label == c).squeeze(), c - 1] = 1
return x
def __len__(self):
return int(np.floor(len(self._ids) / self._batch_size)) * self._repeats
def __getitem__(self, item):
indexes = self._indexes[item * self._batch_size:(item + 1) *
self._batch_size]
ids_temp = [self._ids[idx] for idx in indexes]
X = [
cv2.resize(
cv2.imread(os.path.join(self._features, ids_temp[k][0]),
cv2.IMREAD_COLOR), self._image_size,
cv2.INTER_NEAREST).astype(np.float64) / 255.0
for k in range(len(ids_temp))
]
y = [
cv2.resize(
cv2.imread(os.path.join(self._targets, ids_temp[k][1]),
cv2.IMREAD_COLOR), self._image_size,
cv2.INTER_NEAREST) for k in range(len(ids_temp))
]
params = self._augmenter.get_random_transform(self._image_size)
if not self._verification:
X = np.array([
self._augmenter.apply_transform(self._augmenter.standardize(x),
params) for x in X
])
y = np.array([
self._encode_one_hot(
cv2.resize(
self._augmenter.apply_transform(_y, params)[:, :, 0],
self._label_size, cv2.INTER_NEAREST).reshape(-1, 1))
for _y in y
])
else:
X = np.array([self._augmenter.standardize(x) for x in X])
y = np.array([
self._encode_one_hot(
cv2.resize(_y[:, :, 0], self._label_size,
cv2.INTER_NEAREST).reshape(-1, 1)) for _y in y
])
return np.array(X), np.array(y)
def on_epoch_end(self):
self._indexes = np.arange(len(self._ids))
if self._shuffle:
np.random.shuffle(self._indexes)
self._indexes = np.repeat(self._indexes, self._repeats)
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self,
list_IDs,
labels,
batch_size=32,
dim=(32, 32),
n_channels=1,
n_sequence=4,
shuffle=True,
path_dataset=None,
type_gen='train',
option=None):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_sequence = n_sequence
self.shuffle = shuffle
self.path_dataset = path_dataset
self.type_gen = type_gen
self.option = option
self.aug_gen = ImageDataGenerator()
print("all:", len(self.list_IDs), " batch per epoch",
int(np.floor(len(self.list_IDs) / self.batch_size)))
self.on_epoch_end()
self.n = 0
self.max = self.__len__()
def __next__(self):
if self.n >= self.max:
self.n = 0
result = self.__getitem__(self.n)
self.n += 1
return result
def __len__(self):
return int(np.floor(len(self.list_IDs) / self.batch_size))
def on_epoch_end(self):
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
X, y = self.__data_generation(list_IDs_temp)
if self.type_gen == 'predict':
return X
else:
return X, y
def get_sampling_frame(self, len_frames):
return range(self.n_sequence)
# return np.linspace(1, len_frames-2, self.n_sequence, dtype='int') # last frame gives error sometimes, skip it
def sequence_augment(self, sequence):
name_list = [
'rotate', 'width_shift', 'height_shift', 'brightness',
'flip_horizontal', 'width_zoom', 'height_zoom'
]
dictkey_list = [
'theta', 'ty', 'tx', 'brightness', 'flip_horizontal', 'zy', 'zx'
]
random_aug = np.random.randint(2, 5) # random 2-4 augmentation method
pick_idx = np.random.choice(len(dictkey_list),
random_aug,
replace=False) #
dict_input = {}
for i in pick_idx:
if dictkey_list[i] == 'theta':
dict_input['theta'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'ty': # width_shift
dict_input['ty'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'tx': # height_shift
dict_input['tx'] = np.random.randint(-10, 10)
elif dictkey_list[i] == 'brightness':
dict_input['brightness'] = np.random.uniform(0.8, 1.2)
elif dictkey_list[i] == 'flip_horizontal':
dict_input['flip_horizontal'] = True
elif dictkey_list[i] == 'zy': # width_zoom
dict_input['zy'] = np.random.uniform(0.75, 1.25)
elif dictkey_list[i] == 'zx': # height_zoom
dict_input['zx'] = np.random.uniform(0.75, 1.25)
len_seq = sequence.shape[0]
for i in range(len_seq):
sequence[i] = self.aug_gen.apply_transform(sequence[i], dict_input)
return sequence
def albu_aug(self, sequence, tfms=albu_tfms):
seed = random.randint(0, 99999)
for index, x in enumerate(sequence):
random.seed(seed)
sequence[index] = tfms(image=x.astype('uint8'))['image']
return sequence
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples'
# Initialization
if self.option == "Fuse":
X = np.empty((self.batch_size, self.n_sequence * 2 - 1, *self.dim,
self.n_channels)) # (bs, 127, 224, 224, 3)
elif self.option == "Flow":
X = np.empty((self.batch_size, self.n_sequence - 1, *self.dim,
self.n_channels)) # (bs, 63, 224, 224, 3)
else:
X = np.empty((self.batch_size, self.n_sequence, *self.dim,
self.n_channels)) # (bs, 64, 224, 224, 3)
Y = np.empty((self.batch_size), dtype=int)
for i, ID in enumerate(list_IDs_temp): # ID is name of file
path_file = self.path_dataset + ID + '.avi'
cap = cv2.VideoCapture(path_file)
length_file = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)
) # get how many frames this video have
index_sampling = self.get_sampling_frame(
length_file) # get sampling index, returns 64 index
if self.option == "Flow":
cap.set(cv2.CAP_PROP_POS_FRAMES, 1)
for j, n_pic in enumerate(range(self.n_sequence)):
# cap.set(cv2.CAP_PROP_POS_FRAMES, n_pic) # jump to that index
ret, frame = cap.read()
try:
frame = cv2.cvtColor(frame,
cv2.COLOR_BGR2RGB) # forgot to do it
frame = cv2.resize(frame, self.dim[::-1])
except:
print(frame, length_file, path_file, "Using old image")
X[i, j, :, :, :] = frame
if self.option != 'RGB': # for fuse and flow
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
ret, frame = cap.read()
frame = cv2.cvtColor(frame,
cv2.COLOR_BGR2RGB) # forgot to do it
frame = cv2.resize(frame, self.dim[::-1])
temp = (X[i, ]).astype('uint8')
temp = np.insert(temp, 0, frame, axis=0)
if self.option == 'Flow':
X[i, ] = compute_flow(temp)
elif self.option == 'Fuse': # using both RGB + Flow => X will have shape of (BS, 2*n_sequnece, 224, 224, 3)
X[i, ] = np.concatenate(
(X[i, :self.n_sequence],
compute_flow(temp[:self.n_sequence])),
axis=0)
# augmentation and retrieving labels
if self.type_gen == 'train':
X[i, ] = (self.sequence_augment(
X[i, ]
)) / 255.0 # each sample undergoes the same transformation
else:
X[i, ] = X[i, ] / 255.0
Y[i] = self.labels[ID]
cap.release()
if self.option == "Fuse":
return [
X[:, :self.n_sequence, :, :, :], X[:,
self.n_sequence:, :, :, :]
], Y
else:
if self.option == "Grey":
X = X * 255
X = X.astype('uint8')
X = create_outline(X)
return X, Y
def AugVal_reg(in_vol_in, in_vol_out, num_out, minmax):
new_vol_in = np.empty(
[in_vol_in.shape[0], in_vol_in.shape[1], in_vol_in.shape[2], 1])
new_vol_out = np.zeros((num_out, 1))
sh = new_vol_in.shape
Vols = [[], []]
datagen = ImageDataGenerator()
r_list = [-0.5, 0.25, 0.11]
flip_list = [False, False, True]
for i in range(len(r_list)):
# prepare i different transformations
r = r_list[i]
flip = flip_list[i]
# print("r= ", r)
imageData = np.empty([sh[0], sh[1], sh[2]])
image = np.zeros([sh[0], sh[1], 1])
for j in range(sh[2]):
# apply transformation on image
image1 = np.expand_dims(in_vol_in[:, :, j], 2)
image = datagen.apply_transform(
image1,
transform_parameters={
'flip_vertical': flip,
# vertical flip = upside down image
'tx': r * 10, # vertical translation
'ty': r * 10
}) # horizontal translation
# Do preprocess on input images and change gray level of images to [0 255] and eliminate
# nan and -inf values from image pixels value and active below process
image = np.squeeze(image, axis=2)
# image = np.uint8(image)
# image = image / 255
# image[image < 0] = 0
# image[image > 1] = 1
imageData[:, :, j] = image # (128, 128, 99)
imageData = imageData.reshape(sh[0], sh[1], sh[2], 1)
new_vol_in[:, :, :, :] = imageData
Vols[0].append(new_vol_in)
# Add original image in Vols[0]/ After preprocess of input images , active below process
# in_vol_in = np.uint8(in_vol_in)
# in_vol_in = in_vol_in / 255
# in_vol_in[in_vol_in < 0] = 0
# in_vol_in[in_vol_in > 1] = 1
in_vol_in = in_vol_in.reshape(sh[0], sh[1], sh[2], 1)
Vols[0].append(in_vol_in)
# Normalized coefficients with total min and total max of all coefficients
new_vol_out = (in_vol_out - minmax[0]) / (minmax[1] - minmax[0])
Vols[1].append(new_vol_out[:, 1])
Vols[1].append(new_vol_out[:, 1])
Vols[1].append(new_vol_out[:, 1])
Vols[1].append(new_vol_out[:, 1])
# Vols.append(new_vol_in)
# Vols.append(new_vol_out)
return Vols
class HPatches():
def __init__(self,
train=True,
transform=None,
download=False,
train_fnames=[],
test_fnames=[],
denoise_model=None,
use_clean=False):
self.train = train
self.transform = transform
self.train_fnames = train_fnames
self.test_fnames = test_fnames
self.denoise_model = denoise_model
self.use_clean = use_clean
self.imgaug = ImageDataGenerator(
# featurewise_center=True,
# featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
def set_denoise_model(self, denoise_model):
self.denoise_model = denoise_model
def denoise_patches(self, patches):
batch_size = 100
for i in tqdm(range(int(len(patches) / batch_size)), file=sys.stdout):
batch = patches[i * batch_size:(i + 1) * batch_size]
batch = np.expand_dims(batch, -1)
batch = np.clip(
self.denoise_model.predict(batch).astype(int), 0,
255).astype(np.uint8)[:, :, :, 0]
patches[i * batch_size:(i + 1) * batch_size] = batch
batch = patches[i * batch_size:]
batch = np.expand_dims(batch, -1)
batch = np.clip(self.denoise_model.predict(batch).astype(int), 0,
255).astype(np.uint8)[:, :, :, 0]
patches[i * batch_size:] = batch
return patches
def read_image_file(self, data_dir, train=1):
"""Return a Tensor containing the patches
"""
if self.denoise_model and not self.use_clean:
print('Using denoised patches')
elif not self.denoise_model and not self.use_clean:
print('Using noisy patches')
elif self.use_clean:
print('Using clean patches')
sys.stdout.flush()
patches = []
labels = []
counter = 0
hpatches_sequences = [x[1] for x in os.walk(data_dir)][0]
if train:
list_dirs = self.train_fnames
else:
list_dirs = self.test_fnames
for directory in tqdm(hpatches_sequences, file=sys.stdout):
if (directory in list_dirs):
for tp in tps:
if self.use_clean:
sequence_path = os.path.join(data_dir, directory,
tp) + '.png'
else:
sequence_path = os.path.join(data_dir, directory,
tp) + '_noise.png'
image = cv2.imread(sequence_path, 0)
h, w = image.shape
n_patches = int(h / w)
n_aug = 3
for i in range(n_patches):
patch = image[i * (w):(i + 1) * (w), 0:w]
patch = cv2.resize(patch, (32, 32))
patch = np.array(patch, dtype=np.uint8)
patches.append(patch)
labels.append(i + n_aug * i + counter)
for j in range(0, n_aug):
params = self.imgaug.get_random_transform(
patch.shape)
patch_aug = self.imgaug.apply_transform(
self.imgaug.standardize(patch), params)
patches.append(patch_aug)
labels.append(i + j + counter)
counter += n_patches
patches = np.array(patches, dtype=np.uint8)
if self.denoise_model and not self.use_clean:
print('Denoising patches...')
patches = self.denoise_patches(patches)
return patches, labels
class DataGenerator(tensorflow.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self,
data_type='train',
dim=(84, 84),
n_channels=3,
way=5,
shot=1,
query=5,
num_batch=500):
'Initialization'
self.type = data_type
# if self.type == 'train':
# self.is_training = np.array([True for _ in range(batch_size)])
# else:
# self.is_training = np.array([False for _ in range(batch_size)])
self.dim = dim
#self.batch_size = batch_size
self.n_channels = n_channels
self.num_per_class = 600
self.num_batch = num_batch
#self.y_target = np.zeros(self.batch_size)
self.build_data(self.type)
self.on_epoch_end()
self.way = way
self.shot = shot
self.query = query
self.transformer = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2,
rotation_range=30,
horizontal_flip=True,
shear_range=0.1)
#TODO!!!!
#self.hard_batch = np.zeros(batch_size, *dim, n_channels)
def build_data(self, data_type):
if data_type == 'train':
self.class_data = np.load('python/mini_train.npy')
elif data_type == 'val':
self.class_data = np.load('python/mini_val.npy')
else:
self.class_data = np.load('python/mini_test.npy')
self.n_classes = len(self.class_data)
def __len__(self):
'Denotes the number of batches per epoch'
return self.num_batch
def __getitem__(self, index):
'Generate one batch of data'
# Generate data
X_sample, X_query, label = self.__data_generation()
#way = np.ones((self.way * self.shot, 1)) * self.way
return [X_sample, X_query], label
def on_epoch_end(self):
'Updates indexes after each epoch'
pass
def __data_generation(self):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X_sample = np.empty((self.way, self.shot, *self.dim, self.n_channels))
X_query = np.empty((self.way, self.query, *self.dim, self.n_channels))
chosen_class = random.sample(range(self.n_classes), self.way)
label = np.empty(self.way * self.query)
# print(pos, neg)
# print(self.class_data[pos][0].shape)
# Generate data
for i in range(self.way):
sample_idx = random.sample(range(self.num_per_class),
self.shot + self.query)
sample_data = self.class_data[chosen_class[i]][sample_idx] / 255.
if True:
#if self.type != 'train':
X_sample[i] = sample_data[:self.shot]
X_query[i] = sample_data[self.shot:self.shot + self.query]
else:
for j in range(self.shot):
params = self.transformer.get_random_transform(
self.dim + (self.n_channels, ))
x = self.transformer.apply_transform(
sample_data[j], params)
X_sample[i][j] = x
for j in range(self.shot, self.shot + self.query):
params = self.transformer.get_random_transform(
self.dim + (self.n_channels, ))
x = self.transformer.apply_transform(
sample_data[j], params)
X_query[i][j - self.shot] = x
label[i * self.query:(i + 1) * self.query] = i
return X_sample, X_query, np_utils.to_categorical(label)
def convert_nyu(path):
if not os.path.isfile(path):
print('File not exist: %s, starting download NYU dataset' %
NYU_FILE_PATH)
filename = wget.download(NYU_FILE_URL, out="data")
print('\nDownloaded: ', filename)
print('Loading dataset: %s' % (path))
h5file = h5py.File(path, 'r')
# # print all mat variabls:
# # (u'depths', <HDF5 dataset "depths": shape (1449, 640, 480), type "<f4">)
# # (u'images', <HDF5 dataset "images": shape (1449, 3, 640, 480), type "|u1">)
# for item in h5file.items():
# print(str(item))
# image = h5file['images'][0]
# training example contain a list of rgb depth pairs.
train_examples = []
dev_examples = []
if not os.path.isdir(TRAIN_FILE_PATH):
os.mkdir(TRAIN_FILE_PATH)
file_count = h5file['images'].shape[0]
train_file_count = file_count * (1.0 - DEV_PERCENT)
datagen = ImageDataGenerator()
for i in range(file_count):
image = np.transpose(h5file['images'][i], (2, 1, 0))
depth = np.transpose(h5file['depths'][i], (1, 0))
image_name = os.path.join(TRAIN_FILE_PATH, '%05d_c.png' % (i))
depth_name = os.path.join(TRAIN_FILE_PATH, '%05d_d.png' % (i))
# save to local png file.
if not os.path.isfile(image_name):
image_im = Image.fromarray(np.uint8(image))
image_im.save(image_name)
if not os.path.isfile(depth_name):
scaled_depth = (depth / np.max(depth)) * 255.0
depth_im = Image.fromarray(np.uint8(scaled_depth))
depth_im.save(depth_name)
for augment_count in range(AUGMENTATION_COUNT):
brightness = random.uniform(0.7, 1.0)
zoom_scale = random.uniform(0.7, 1.0)
flip_horizontal = bool(random.getrandbits(1))
augmented_im_bytes = datagen.apply_transform(x=image,
transform_parameters={
'brightness':
brightness,
'zx':
zoom_scale,
'zy':
zoom_scale,
'flip_horizontal':
flip_horizontal
})
augmented_im = Image.fromarray(np.uint8(augmented_im_bytes))
agumented_image_name = os.path.join(
TRAIN_FILE_PATH, '%05d_c_aug_%d.png' % (i, augment_count))
augmented_im.save(agumented_image_name)
# expand depth to 3 channels, keras apply_transform can only tranform 3 channel images.
depth_multi_channel = np.array([depth, depth, depth])
# tranpose depth image to (height, width, channel)
depth_multi_channel = np.transpose(depth_multi_channel, (1, 2, 0))
augmented_depth_bytes = datagen.apply_transform(
x=depth_multi_channel,
transform_parameters={
'zx': zoom_scale,
'zy': zoom_scale,
'flip_horizontal': flip_horizontal
})
# get back single channel depth
single_channel_aug_d = augmented_depth_bytes[:, :, 0]
single_channel_aug_d = (single_channel_aug_d /
np.max(single_channel_aug_d)) * 255.0
augmented_depth = Image.fromarray(np.uint8(single_channel_aug_d))
agumented_depth_name = os.path.join(
TRAIN_FILE_PATH, '%05d_d_aug_%d.png' % (i, augment_count))
augmented_depth.save(agumented_depth_name)
train_examples.append((agumented_image_name, agumented_depth_name))
if i < train_file_count:
train_examples.append((image_name, depth_name))
else:
dev_examples.append((image_name, depth_name))
print('Processed file: %i out of %d' % (i, file_count))
random.shuffle(train_examples)
# write train_examples to csv
with open('data/train.csv', 'w') as output:
for (image_name, depth_name) in train_examples:
output.write("%s,%s" % (image_name, depth_name))
output.write("\n")
with open('data/dev.csv', 'w') as output:
for (image_name, depth_name) in dev_examples:
output.write("%s,%s" % (image_name, depth_name))
output.write("\n")
