def get_cifar_gen():
# get dataset
(x_train, y_train), (x_test,
y_test) = tf.keras.datasets.cifar100.load_data(
label_mode='fine')
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255.
x_test /= 255.
y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)
print('x_train shape:', x_train.shape)
print('y_train shape:', y_train.shape)
# preprocess data
x_train, x_test = pre_processing(x_train, x_test)
datagen = ImageDataGenerator(shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
cifar_gen = datagen.flow(x_train, y_train, batch_size=batch_size)
testgen = ImageDataGenerator()
cifar_test_gen = testgen.flow(x_test, y_test, batch_size=batch_size)
return cifar_gen, cifar_test_gen
def __init__(self, batch_size=64, data_aug=True):
self.batch_size = batch_size
(self.x_train,
self.y_train), (self.x_valid,
self.y_valid) = tf.keras.datasets.mnist.load_data()
self.x_train = self.x_train / 127.5 - 1
self.x_valid = self.x_valid / 127.5 - 1
self.train_size = len(self.x_train)
self.valid_size = len(self.x_valid)
self.img_shape = self.x_train[0].shape
self.x_train = self.x_train.reshape(self.train_size, self.img_shape[0],
self.img_shape[1], 1)
self.x_valid = self.x_valid.reshape(self.valid_size, self.img_shape[0],
self.img_shape[1], 1)
if data_aug:
_train_gen = ImageDataGenerator(rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2)
else:
_train_gen = ImageDataGenerator()
_valid_gen = ImageDataGenerator()
self.train_img_gen = _train_gen.flow(self.x_train,
self.y_train,
batch_size=self.batch_size)
self.valid_img_gen = _valid_gen.flow(self.x_valid,
self.y_valid,
batch_size=self.batch_size)
def augmentation_viewer():
training_file = 'data/train.p'
with open(training_file, mode='rb') as f:
train = pickle.load(f)
image = train['features'][3432]
cv2.imwrite('project_report/before_image.png',
cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
data_augmentor = ImageDataGenerator(
rotation_range=15,
zoom_range=.15,
width_shift_range=.1,
height_shift_range=.1,
shear_range=.15,
horizontal_flip=False,
vertical_flip=False,
fill_mode='nearest',
brightness_range=[.2, 1],
)
data_augmentor.flow(image.reshape([1, 32, 32, 3]),
batch_size=1,
save_to_dir='project_report',
save_prefix="image",
save_format="jpg")
plt.show()
def create_custom_iterators(cls, data, labels, valid_split):
#print(f"The labels are {labels} and their dims are {labels.size}")
if data.shape == 0:
data = tf.constant([], shape=(0, 0, 0, 0))
if labels.shape != 0: # the list of labels is not empty
scalar_classes = tf.math.argmax(labels, axis=1)
#print(
# f"The scalar classes are {scalar_classes} and their dims are {scalar_classes.shape}")
labels = to_categorical(scalar_classes,
num_classes=cls.get_default_num_classes())
else:
labels = tf.constant([], shape=(0, 0, 0, 0))
# print(f"The newly structured labels are {labels} and their dims are {labels.size}")
train_datagen = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
validation_split=valid_split)
valid_datagen = ImageDataGenerator(validation_split=valid_split)
train_iter = train_datagen.flow(data, labels,
batch_size=cls._batch_size,
shuffle=True, seed=123,
subset="training")
valid_iter = valid_datagen.flow(data, labels,
batch_size=cls._batch_size,
shuffle=True, seed=123,
subset="validation")
return train_iter, valid_iter
def augment_data(x_train, y_train, x_val, y_val, batch_size=64):
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
val_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
train_datagen.fit(x_train)
train_generator = train_datagen.flow(x_train,
y_train,
batch_size=batch_size)
val_datagen.fit(x_val)
val_generator = val_datagen.flow(x_val, y_val, batch_size=batch_size)
return train_generator, val_generator
def fit(model, X_train, y_train, X_test, y_test, epochs, batch_size):
img_gen = ImageDataGenerator()
return model.fit_generator(
img_gen.flow(X_train, y_train),
steps_per_epoch=len(X_train) // batch_size,
epochs=epochs,
validation_data=img_gen.flow(X_test, y_test),
validation_steps=len(X_test) // batch_size,
verbose=1,
)
def tta_prediction(model, image, n_examples):
"""
make a prediction for one image using test-time augmentation
@param model: model for prediction
@param image: input image to be predicted (nrows* ncols * nchns)
@param n_exampls:
@return yhat: predicted label for image (scalar int)
"""
# configure image data augmentation
datagen = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
data_format='channels_last',
# rotation_range=15,
)
samples = np.expand_dims(image, 0)
it = datagen.flow(samples, batch_size=n_examples)
yhats = model.predict_generator(it, verbose=0)
summed = np.sum(yhats, axis=0)
yhat = np.argmax(summed)
print(yhat)
print(yhats)
return yhat
def augment_image(self, filename, number_mult):
"""
applies ImageDataGenerator and generate given number of randomly
created images from the base one, which has the filename path
"""
# extract the path to the folder
folder = '/'.join(filename.split('/')[:-1])
# read the image into a numpy array
image = np.expand_dims(imread(str(filename)), 0)
# create datagenetator
datagen = ImageDataGenerator(
rotation_range=10,
zoom_range=0.1,
brightness_range=[0.1, 1],
width_shift_range=0.1,
height_shift_range=0.1
)
datagen.fit(image)
for x, val in zip(datagen.flow(image, # image we chose
save_to_dir=folder, # this is where we figure out where to save
save_prefix='aug',
# it will save the images as 'aug_0912' some number for every new augmented image
save_format='png'), range(
number_mult)): # here we define a range because we want 10 augmented images otherwise it will keep looping forever I think
pass
def get_slice_generator(path: str = 'data/im_data.pickle',
slice_per_file: int = 128,
slice_shape=(224, 224, 1),
val_split: float = 0.2,
batch_size: int = 32) -> tuple:
# Load dataset from .pickle file
dataset = load_dataset(path)
# Extract slice from 3D img
X, Y = generate_slice_dataset(dataset, slice_per_file, slice_shape)
# Shuffle arrays
p = np.random.permutation(len(X))
X, Y = X[p], Y[p]
# Extract validation data
size_valid = int(len(X) * val_split)
X_valid, Y_valid = X[0:size_valid], Y[0:size_valid]
X_train, Y_train = X[size_valid:len(X)], Y[size_valid:len(Y)]
# Create generator
datagen = ImageDataGenerator(width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.05,
zoom_range=0.05,
dtype=np.float16)
datagen.fit(X)
data_generator = datagen.flow(X_train, Y_train, batch_size)
data_validation = (X_valid, Y_valid)
x_train_size = len(X_train)
return data_generator, data_validation, x_train_size
def generator(self,fp,imgs,path):
# 이미지 증식
idg = ImageDataGenerator(
# rescale=1/255, # 스케일 변경
rotation_range=40.0, # 회전 각도
width_shift_range=0.2, # 수평 방향 이동 비율
height_shift_range=0.2, # 수직 방향 이동 비율
shear_range=0.2, # 반시계 방향의 전단 강도(radian)
zoom_range=0.2, # 랜덤하게 확대할 사진의 범위 지정
horizontal_flip=True, # 수평 방향으로 입력 반전
fill_mode = 'nearest'
# vertical_flip=True # 수직 방향으로 입력 반전
)
class_name = fp.split('\\')[-1]
for img_fp in imgs:
img = load_img(fp+'\\'+img_fp)
ary = img_to_array(img)
ary = ary.reshape((1,)+ary.shape)
i = 0
for batch in idg.flow(ary, batch_size=1,
save_to_dir=f'../../data/crl_image/{path}/{class_name}',
save_prefix=class_name,save_format='jpeg'):
i += 1
if i > 20: break
print(fp,imgs)
def createAugmentedData(training_data, training_labels):
"""
This is creates the augmented data.
Args:
training_data(numpy arrays): This is the numpy array of the training data.
training_labels(numpy arrays): This is the numpy array of the training labels.
Returns:
complete_training_data_set(numpy array): This is the numpy array of the total training data, which is has
undergone augmentation.
complete_training_labels_set(numpy array): This is the numpy array of the total training labels, which is has
undergone augmentation.
"""
complete_training_data_set = []
complete_training_labels_set = []
for data in training_data:
complete_training_data_set.append(data)
print("Complete Training Data: " + str(len(complete_training_data_set)))
for label in training_labels:
complete_training_labels_set.append(label)
print("Complete Training Label: " + str(len(complete_training_labels_set)))
# create augmented data
data_augmented = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=90,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
vertical_flip=True)
data_augmented.fit(training_data)
training_data_size = training_data.shape[0]
aug_counter = 0
while aug_counter < (augmented_multiple - 1):
iterator = data_augmented.flow(training_data,
training_labels,
batch_size=training_data_size)
# iterator = data_augmented.flow(training_data, training_labels, batch_size=batch_size)
augmented_data = iterator.next()
for data in augmented_data[0]:
complete_training_data_set.append(data)
for label in augmented_data[1]:
complete_training_labels_set.append(label)
aug_counter += 1
print("Size of All Training Data: " + str(len(complete_training_data_set)))
print("Size of All Training Labels: " +
str(len(complete_training_labels_set)))
array_training_data = np.array(complete_training_data_set)
array_training_labels = np.array(complete_training_labels_set)
print("Shape of complete training data: " + str(array_training_data.shape))
print("Shape of complete training labels: " +
str(array_training_labels.shape))
return np.array(complete_training_data_set), np.array(
complete_training_labels_set)
def _train(self):
x_train, y_train = self.train_data
x_test, y_test = self.test_data
aug_gen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False, # randomly flip images
)
aug_gen.fit(x_train)
gen = aug_gen.flow(x_train,
y_train,
batch_size=self.config['batch_size'])
self.model.fit_generator(generator=gen,
steps_per_epoch=50000 //
self.config['batch_size'],
epochs=self.config['epochs'],
validation_data=None)
# loss, accuracy
_, accuracy = self.model.evaluate(x_test, y_test, verbose=0)
return TrainingResult(timesteps_this_iter=10, mean_accuracy=accuracy)
def main(net, epochs, batch_size):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train.astype('float32') / 255, x_test.astype(
'float32') / 255
mean = np.mean(x_train, axis=0)
x_train -= mean
x_test -= mean
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_train)
model = make_resnet(net)
model.summary()
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_test, y_test),
epochs=epochs,
callbacks=[
ReduceLROnPlateau(verbose=1, patience=20),
TensorBoard(observer.dir)
])
def data_augmentation(in_dir, out_dir, result_images):
images_list = [os.path.join(in_dir, f) for f in os.listdir(in_dir)]
num_images_from_one = math.ceil(result_images / len(images_list))
aug = ImageDataGenerator(rotation_range=30,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest")
for (image_num, image_file) in enumerate(images_list):
image = load_img(image_file)
image = img_to_array(image)
image = np.expand_dims(image, axis=0)
imageGen = aug.flow(image,
batch_size=1,
save_to_dir=out_dir,
save_prefix="image" + str(image_num),
save_format="jpg")
total = 0
for image in imageGen:
total += 1
if total == num_images_from_one:
break
def evaluate(model, X_train, y_train):
img_gen = ImageDataGenerator()
# evaluate the model
scores = model.evaluate_generator(img_gen.flow(X_train, y_train),
workers=4,
verbose=1)
print("%s: %.2f" % (model.metrics_names[1], scores[1]))
def evaluate_on_cifar10():
total_depth = 36
n_blocks = 3
basic_block_count = total_depth // n_blocks
# region Model
input_layer = Input(shape=[32, 32, 3])
layer = input_layer
kernel_initializer = ResBlock2D.get_fixup_initializer(total_depth)
for k in range(n_blocks):
strides = 2 if k < (n_blocks - 1) else 1
layer = ResBlock2D(filters=16 * (2**k),
basic_block_count=basic_block_count,
strides=strides,
kernel_initializer=kernel_initializer,
use_residual_bias=True)(layer)
if k == (n_blocks - 1):
layer = AveragePooling2D(pool_size=8)(layer)
layer = Flatten()(layer)
layer = Dense(units=10, activation="softmax")(layer)
model = Model(inputs=input_layer, outputs=layer)
model.summary()
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["acc"])
# endregion
# region Data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype(np.float32) / 255.0
x_test = x_test.astype(np.float32) / 255.0
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
generator = ImageDataGenerator(rotation_range=15,
width_shift_range=5. / 32,
height_shift_range=5. / 32,
horizontal_flip=True)
generator.fit(x_train, seed=0)
# endregion
log_dir = "../logs/tests/res_block_cifar10/{}".format(int(time()))
log_dir = os.path.normpath(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, profile_batch=0)
model.fit_generator(generator.flow(x_train, y_train, batch_size=64),
steps_per_epoch=100,
epochs=300,
validation_data=(x_test, y_test),
validation_steps=100,
verbose=1,
callbacks=[tensorboard])
def train_generator(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(
width_shift_range=shift_fraction,
height_shift_range=shift_fraction)
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
yield ([x_batch, y_batch], [y_batch, x_batch])
def augment_data(x_train, y_train, x_val, y_val, batch_size=128):
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=15,
horizontal_flip=True)
val_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=15,
horizontal_flip=True)
train_datagen.fit(x_train)
train_generator = train_datagen.flow(x_train,
y_train,
batch_size=batch_size)
val_datagen.fit(x_val)
val_generator = val_datagen.flow(x_val, y_val, batch_size=batch_size)
return train_generator, val_generator
def get_train_generator(self, batch_size):
"""
Returns a batch generator which transforms chunk of raw images into numpy matrices
and then "yield" them for the classifier. Doing so allow to greatly optimize
memory usage as the images are processed then deleted by chunks (defined by batch_size)
instead of preprocessing them all at once and feeding them to the classifier.
:param batch_size: int
The batch size
:return: generator
The batch generator
"""
# Image Augmentation
datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
horizontal_flip=True,
vertical_flip=True) # randomly flip images horizontally
loop_range = len(self.X_train)
while True:
for i in range(loop_range):
start_offset = batch_size * i
# The last remaining files could be smaller than the batch_size
range_offset = min(batch_size, loop_range - start_offset)
# If we reached the end of the list then we break the loop
if range_offset <= 0:
break
batch_features = np.zeros((range_offset, *self.img_resize, 3))
batch_labels = np.zeros((range_offset, len(self.y_train[0])))
for j in range(range_offset):
# Maybe shuffle the index?
img = Image.open(self.X_train[start_offset + j])
img.thumbnail(self.img_resize)
# Augment the image `img` here
# Convert to RGB and normalize
img_array = np.asarray(img.convert("RGB"),
dtype=np.float32)
img_array = img_array[:, :, ::-1]
# Zero-center by mean pixel
img_array[:, :, 0] -= 103.939
img_array[:, :, 1] -= 116.779
img_array[:, :, 2] -= 123.68
batch_features[j] = img_array
batch_labels[j] = self.y_train[start_offset + j]
# Augment the images (using Keras allow us to add randomization/shuffle to augmented images)
# Here the next batch of the data generator (and only one for this iteration)
# is taken and returned in the yield statement
yield next(
datagen.flow(batch_features, batch_labels, range_offset))
def prepare_training_data(self, images: ndarray,
labels: ndarray) -> Tuple[Any, int]:
train_datagen = ImageDataGenerator()
steps = self.steps if isinstance(
self.steps, int) else len(images) / self.batch_size
return train_datagen.flow(images,
labels,
batch_size=self.batch_size,
shuffle=True), steps
def get_gen(set_name, batch_size, translate, scale, shuffle=True):
if set_name == 'train':
(X, Y), _ = get_mnist_dataset()
elif set_name == 'test':
_, (X, Y) = get_mnist_dataset()
image_gen = ImageDataGenerator(zoom_range=scale,
width_shift_range=translate,
height_shift_range=translate)
gen = image_gen.flow(X, Y, batch_size=batch_size, shuffle=shuffle)
return gen
def image_augmentation(img, label, augnum): #num means batch_size
#label = label.reshape((1,)+label.shape)
image_datagen = ImageDataGenerator(rotation_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
fill_mode='nearest')
label_datagen = ImageDataGenerator(rotation_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
fill_mode='nearest')
seed = random.randint(1, 10000)
n = 0
seed_ = 1
image_datagen.fit(img, seed=seed_)
label_datagen.fit(label, seed=seed_)
for batch in image_datagen.flow(img,
batch_size=1,
save_to_dir=IMG_PATH,
save_prefix='aug',
save_format='png',
seed=seed):
n += 1
if n > augnum:
break
n = 0
for batch in label_datagen.flow(label,
batch_size=1,
save_to_dir=LABEL_PATH,
save_prefix='aug_label',
save_format='png',
seed=seed):
n += 1
if n > augnum:
break
return
'''
def image_augmentation(img, label, augnum): #num means batch_size
image_datagen = ImageDataGenerator(rotation_range=0.2,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1,
fill_mode='nearest')
label_datagen = ImageDataGenerator(rotation_range=0.2,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1,
fill_mode='nearest')
#random.seed(1)
seed = random.randint(1, 100000)
n = 0
for batch in image_datagen.flow(img,
batch_size=1,
save_to_dir=path,
save_prefix='aug',
save_format='png',
seed=seed):
n += 1
if n > augnum:
break
n = 0
for batch in image_datagen.flow(label,
batch_size=1,
save_to_dir=path_,
save_prefix='aug_label',
save_format='png',
seed=seed):
n += 1
if n > augnum:
break
return
def img_and_mask_generator(x, y, batch_size=1, shuffle=True):
"""
Create a generator of two combined ImageDataGenerators for input and ground truth images without any data augmentation except scaling.
"""
data_gen_args = dict(rescale=1. / 255)
image_data_generator = ImageDataGenerator(**data_gen_args)
mask_data_generator = ImageDataGenerator(**data_gen_args)
seed = 1
if isinstance(x, np.ndarray):
image_gen = image_data_generator.flow(x,
batch_size=batch_size,
seed=seed,
shuffle=shuffle)
mask_gen = mask_data_generator.flow(y,
batch_size=batch_size,
seed=seed,
shuffle=shuffle)
else:
image_gen = image_data_generator.flow_from_directory(
x,
batch_size=batch_size,
seed=seed,
shuffle=shuffle,
class_mode=None,
color_mode="grayscale",
target_size=(465, 381))
mask_gen = mask_data_generator.flow_from_directory(
y,
batch_size=batch_size,
seed=seed,
shuffle=shuffle,
class_mode=None,
color_mode="grayscale",
target_size=(465, 381))
return zip(image_gen, mask_gen)
def get_generator(self):
"""get generators for training set and validation set"""
generator = ImageDataGenerator(rotation_range=90,
horizontal_flip=True,
vertical_flip=True,
fill_mode='reflect')
# preprocessing
self.binarize_mask_edge()
self.imgs = self.add_axis(self.imgs, repeat=True)
self.masks = self.add_axis(self.masks)
self.edges = self.add_axis(self.edges)
seed = 66
# split crops
imgs_tr, imgs_val, masks_tr, masks_val, edges_tr, edges_val\
= train_test_split(self.imgs, self.masks, self.edges,
test_size=self.SPLIT_RATE, random_state=seed)
self.num_train = len(imgs_tr)
self.num_val = len(imgs_val)
# feed generator with the corresponding data
gene_img = generator.flow(imgs_tr,
batch_size=self.batch_size,
seed=seed)
gene_mask = generator.flow(masks_tr,
batch_size=self.batch_size,
seed=seed)
gene_edge = generator.flow(edges_tr,
batch_size=self.batch_size,
seed=seed)
out_gene = zip(gene_mask, gene_edge)
train_generator = zip(gene_img, out_gene)
# # Visualize augmented crops for debugging
# for i in range(10):
# img = gene_img.next()[i][:, :, 0]
# mask = gene_mask.next()[i][:, :, 0]
# f1 = plt.subplot(211)
# plt.imshow(img, 'gray')
# f2 = plt.subplot(212)
# plt.imshow(mask, 'gray')
# plt.show()
gene_img = generator.flow(imgs_val,
batch_size=self.batch_size,
seed=seed)
gene_mask = generator.flow(masks_val,
batch_size=self.batch_size,
seed=seed)
gene_edge = generator.flow(edges_val,
batch_size=self.batch_size,
seed=seed)
out_gene = zip(gene_mask, gene_edge)
val_generator = zip(gene_img, out_gene)
return train_generator, val_generator
def fit_and_evaluate(train_x, val_x, train_y, val_y):
model = None
gc.collect()
model = createModel(train_x)
batch_size = 32
epochs = 30
gc.collect()
datagen = ImageDataGenerator(zoom_range = 0.2,horizontal_flip = False)
datagen.fit(train_x)
gc.collect()
results = model.fit_generator(datagen.flow(train_x,train_y,batch_size=batch_size), epochs = epochs,callbacks=[early_stopping, model_checkpoint],
verbose=1,validation_data = (val_x,val_y))
gc.collect()
print("Val Score: ", model.evaluate(val_x, val_y))
return model, results
def shift(samples, imagePath, augPath, label, iImg, iAug):
print("[INFO] Saving horizontal and vertical shifting images...")
datagen = ImageDataGenerator(width_shift_range=[-200, 200])
# prepare iterator
it = datagen.flow(samples, batch_size=1)
# generate samples
for i in range(6):
# generate batch of images
batch = it.next()
# convert to unsigned integers for viewing
newImage = batch[0].astype('uint8')
saveImage(imagePath, newImage, augPath, label, iImg, iAug)
iAug += 1
datagen = ImageDataGenerator(height_shift_range=0.5)
# prepare iterator
it = datagen.flow(samples, batch_size=1)
# generate samples
for i in range(6):
# generate batch of images
batch = it.next()
# convert to unsigned integers for viewing
newImage = batch[0].astype('uint8')
saveImage(imagePath, newImage, augPath, l, iImg, iAug)
iAug += 1
return iAug, iImg
def classify(model, img_dir):
X, all_files = load_data_to_test(img_dir)
total = len(all_files)
ans = {}
test_datagen = ImageDataGenerator(rescale=1. / 255)
test_generator = test_datagen.flow(X,
batch_size=IMG_BATCHS_NUMBER,
shuffle=None,
save_to_dir=None)
pred = model.predict_generator(test_generator,
steps=total // IMG_BATCHS_NUMBER)
for i in range(0, pred.shape[0], 1):
print(pred[i])
ans[all_files[i]] = np.argmax(pred[i])
print(ans)
return ans
def fit_and_evaluate(X_train,y_train):
model = None
gc.collect()
model = createModel(X_train)
batch_size = 32
epochs = 30
gc.collect()
datagen = ImageDataGenerator(zoom_range = 0.2,horizontal_flip = False)
datagen.fit(X_train)
gc.collect()
train_x, val_x, train_y, val_y = train_test_split(X_train, y_train, test_size=0.1, random_state = np.random.randint(1,1000, 1)[0])
plotCategories(train_y,val_y)
results = model.fit_generator(datagen.flow(train_x,train_y,batch_size=batch_size), epochs = epochs,steps_per_epoch = X_train.shape[0] // batch_size ,callbacks=[early_stopping, model_checkpoint],
verbose=1,validation_data = (val_x,val_y))
gc.collect()
print("Val Score: ", model.evaluate(val_x, val_y))
return
def train_model(model, x, y, task):
date = datetime.now().strftime("%d-%m-%Y-%H-%M-%S")
checkpoints_path = f'/opt/ml/model/checkpoints/{task}/{date}'
checkpoints_file_format = 'checkpoints.{epoch:02d}-{val_loss:.2f}.hdf5'
checkpoint_callback = ModelCheckpoint(
posixpath.join(checkpoints_path, checkpoints_file_format))
L = len(x)
split_ration = 0.8
train_indexes = random.sample(range(L), int(L * split_ration))
validation_indexes = list(set(range(L)) - set(train_indexes))
x_train = x[train_indexes]
y_train = y[train_indexes]
x_val = x[validation_indexes]
y_val = y[validation_indexes]
generator = ImageDataGenerator(rotation_range=30,
zoom_range=0.20,
fill_mode="nearest",
shear_range=0.20,
horizontal_flip=False,
width_shift_range=0.2,
height_shift_range=0.2)
lr_schedule = ExponentialDecay(initial_learning_rate=0.01,
decay_steps=100,
decay_rate=0.9,
staircase=False)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(learning_rate=lr_schedule),
metrics=['accuracy'])
model.fit(generator.flow(x_train, y_train, batch_size=32),
steps_per_epoch=len(x_train) // 32,
verbose=2,
epochs=500,
shuffle=True,
validation_data=(x_val, y_val),
callbacks=[
WandbCallback(), checkpoint_callback,
LearningRateCallback(model)
])
model.save('/opt/ml/model')
def _train(self):
x_train, y_train = self.train_data
x_test, y_test = self.test_data
aug_gen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by dataset std
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# randomly rotate images in the range (degrees, 0 to 180)
rotation_range=0,
# randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
)
aug_gen.fit(x_train)
gen = aug_gen.flow(
x_train, y_train, batch_size=self.config["batch_size"])
self.model.fit_generator(
generator=gen,
steps_per_epoch=50000 // self.config["batch_size"],
epochs=self.config["epochs"],
validation_data=None)
# loss, accuracy
_, accuracy = self.model.evaluate(x_test, y_test, verbose=0)
return {"mean_accuracy": accuracy}
