def main(net, epochs, batch_size):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train.astype('float32') / 255, x_test.astype(
'float32') / 255
mean = np.mean(x_train, axis=0)
x_train -= mean
x_test -= mean
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_train)
model = make_resnet(net)
model.summary()
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_test, y_test),
epochs=epochs,
callbacks=[
ReduceLROnPlateau(verbose=1, patience=20),
TensorBoard(observer.dir)
])
def _train(self):
x_train, y_train = self.train_data
x_test, y_test = self.test_data
aug_gen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False, # randomly flip images
)
aug_gen.fit(x_train)
gen = aug_gen.flow(x_train,
y_train,
batch_size=self.config['batch_size'])
self.model.fit_generator(generator=gen,
steps_per_epoch=50000 //
self.config['batch_size'],
epochs=self.config['epochs'],
validation_data=None)
# loss, accuracy
_, accuracy = self.model.evaluate(x_test, y_test, verbose=0)
return TrainingResult(timesteps_this_iter=10, mean_accuracy=accuracy)
def augment_data(x_train, y_train, x_val, y_val, batch_size=64):
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
val_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
train_datagen.fit(x_train)
train_generator = train_datagen.flow(x_train,
y_train,
batch_size=batch_size)
val_datagen.fit(x_val)
val_generator = val_datagen.flow(x_val, y_val, batch_size=batch_size)
return train_generator, val_generator
def main(net):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train, x_test = x_train.astype('float32') / 255, x_test.astype('float32') / 255
y_train, y_test = y_train.astype('int32'), y_test.astype('int32')
mean = np.mean(x_train, axis=0)
x_train -= mean
x_test -= mean
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_train)
resolver = tf.contrib.cluster_resolver.TPUClusterResolver('matthew-rahtz')
tf.contrib.distribute.initialize_tpu_system(resolver)
strategy = tf.contrib.distribute.TPUStrategy(resolver)
with strategy.scope():
model = make_resnet(net)
model.summary()
# model.fit_generator(datagen.flow(x_train, y_train, batch_size=32),
# validation_data=(x_test, y_test),
# epochs=200,
# callbacks=[ReduceLROnPlateau(verbose=1),
# TensorBoard(observer.dir)])
model.fit(x_train, y_train, batch_size=32, epochs=200, steps_per_epoch=390)
def createAugmentedData(training_data, training_labels):
"""
This is creates the augmented data.
Args:
training_data(numpy arrays): This is the numpy array of the training data.
training_labels(numpy arrays): This is the numpy array of the training labels.
Returns:
complete_training_data_set(numpy array): This is the numpy array of the total training data, which is has
undergone augmentation.
complete_training_labels_set(numpy array): This is the numpy array of the total training labels, which is has
undergone augmentation.
"""
complete_training_data_set = []
complete_training_labels_set = []
for data in training_data:
complete_training_data_set.append(data)
print("Complete Training Data: " + str(len(complete_training_data_set)))
for label in training_labels:
complete_training_labels_set.append(label)
print("Complete Training Label: " + str(len(complete_training_labels_set)))
# create augmented data
data_augmented = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=90,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
vertical_flip=True)
data_augmented.fit(training_data)
training_data_size = training_data.shape[0]
aug_counter = 0
while aug_counter < (augmented_multiple - 1):
iterator = data_augmented.flow(training_data,
training_labels,
batch_size=training_data_size)
# iterator = data_augmented.flow(training_data, training_labels, batch_size=batch_size)
augmented_data = iterator.next()
for data in augmented_data[0]:
complete_training_data_set.append(data)
for label in augmented_data[1]:
complete_training_labels_set.append(label)
aug_counter += 1
print("Size of All Training Data: " + str(len(complete_training_data_set)))
print("Size of All Training Labels: " +
str(len(complete_training_labels_set)))
array_training_data = np.array(complete_training_data_set)
array_training_labels = np.array(complete_training_labels_set)
print("Shape of complete training data: " + str(array_training_data.shape))
print("Shape of complete training labels: " +
str(array_training_labels.shape))
return np.array(complete_training_data_set), np.array(
complete_training_labels_set)
def augment_image(self, filename, number_mult):
"""
applies ImageDataGenerator and generate given number of randomly
created images from the base one, which has the filename path
"""
# extract the path to the folder
folder = '/'.join(filename.split('/')[:-1])
# read the image into a numpy array
image = np.expand_dims(imread(str(filename)), 0)
# create datagenetator
datagen = ImageDataGenerator(
rotation_range=10,
zoom_range=0.1,
brightness_range=[0.1, 1],
width_shift_range=0.1,
height_shift_range=0.1
)
datagen.fit(image)
for x, val in zip(datagen.flow(image, # image we chose
save_to_dir=folder, # this is where we figure out where to save
save_prefix='aug',
# it will save the images as 'aug_0912' some number for every new augmented image
save_format='png'), range(
number_mult)): # here we define a range because we want 10 augmented images otherwise it will keep looping forever I think
pass
def get_slice_generator(path: str = 'data/im_data.pickle',
slice_per_file: int = 128,
slice_shape=(224, 224, 1),
val_split: float = 0.2,
batch_size: int = 32) -> tuple:
# Load dataset from .pickle file
dataset = load_dataset(path)
# Extract slice from 3D img
X, Y = generate_slice_dataset(dataset, slice_per_file, slice_shape)
# Shuffle arrays
p = np.random.permutation(len(X))
X, Y = X[p], Y[p]
# Extract validation data
size_valid = int(len(X) * val_split)
X_valid, Y_valid = X[0:size_valid], Y[0:size_valid]
X_train, Y_train = X[size_valid:len(X)], Y[size_valid:len(Y)]
# Create generator
datagen = ImageDataGenerator(width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.05,
zoom_range=0.05,
dtype=np.float16)
datagen.fit(X)
data_generator = datagen.flow(X_train, Y_train, batch_size)
data_validation = (X_valid, Y_valid)
x_train_size = len(X_train)
return data_generator, data_validation, x_train_size
def evaluate_on_cifar10():
total_depth = 36
n_blocks = 3
basic_block_count = total_depth // n_blocks
# region Model
input_layer = Input(shape=[32, 32, 3])
layer = input_layer
kernel_initializer = ResBlock2D.get_fixup_initializer(total_depth)
for k in range(n_blocks):
strides = 2 if k < (n_blocks - 1) else 1
layer = ResBlock2D(filters=16 * (2**k),
basic_block_count=basic_block_count,
strides=strides,
kernel_initializer=kernel_initializer,
use_residual_bias=True)(layer)
if k == (n_blocks - 1):
layer = AveragePooling2D(pool_size=8)(layer)
layer = Flatten()(layer)
layer = Dense(units=10, activation="softmax")(layer)
model = Model(inputs=input_layer, outputs=layer)
model.summary()
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["acc"])
# endregion
# region Data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype(np.float32) / 255.0
x_test = x_test.astype(np.float32) / 255.0
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
generator = ImageDataGenerator(rotation_range=15,
width_shift_range=5. / 32,
height_shift_range=5. / 32,
horizontal_flip=True)
generator.fit(x_train, seed=0)
# endregion
log_dir = "../logs/tests/res_block_cifar10/{}".format(int(time()))
log_dir = os.path.normpath(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, profile_batch=0)
model.fit_generator(generator.flow(x_train, y_train, batch_size=64),
steps_per_epoch=100,
epochs=300,
validation_data=(x_test, y_test),
validation_steps=100,
verbose=1,
callbacks=[tensorboard])
def augment(X_train):
"""
"""
from tensorflow.python.keras.preprocessing.image import ImageDataGenerator
aug = ImageDataGenerator(rotation_range=20,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest")
aug.fit(X_train)
return aug
def fit_and_evaluate(train_x, val_x, train_y, val_y):
model = None
gc.collect()
model = createModel(train_x)
batch_size = 32
epochs = 30
gc.collect()
datagen = ImageDataGenerator(zoom_range = 0.2,horizontal_flip = False)
datagen.fit(train_x)
gc.collect()
results = model.fit_generator(datagen.flow(train_x,train_y,batch_size=batch_size), epochs = epochs,callbacks=[early_stopping, model_checkpoint],
verbose=1,validation_data = (val_x,val_y))
gc.collect()
print("Val Score: ", model.evaluate(val_x, val_y))
return model, results
def train_model(trainFile, testFile):
train = pd.read_csv(trainFile)
f, ax = plt.subplots(5, 5)
for i in range(1, 26):
data = train.iloc[i, 1:785].values
nrows, ncols = 28, 28
grid = data.reshape((nrows, ncols))
n = math.ceil(i / 5) - 1
m = [0, 1, 2, 3, 4] * 5
ax[m[i - 1], n].imshow(grid)
dataTest = pd.read_csv(testFile)
trainNumbers = train['label']
train = train.drop(labels=['label'], axis=1)
trainNumbers = to_categorical(trainNumbers, num_classes=10)
train = train / 255
test = dataTest / 255
train = train.values.reshape(-1, 28, 28, 1)
test = test.values.reshape(-1, 28, 28, 1)
model = Sequential()
model.add(Conv2D(24, (3, 3), padding='same', input_shape=(28, 28, 1)))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(10, activation='softmax'))
datagen = ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=5,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
datagen.fit(train)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(train, trainNumbers, epochs=5, batch_size=120)
return model
def getDataGen(datagenMode: str = None, dataType=None) -> ImageDataGenerator:
"""
Function to create data generator for training
:param datagenMode: str, 'train' or 'test'
:param dataType: str can be one of these standard datasets
- cifar10
- cifar100
- mnist
- fashion_mnist
or 'custom'
:return: ImageDataGenerator
"""
datagen = ImageDataGenerator(
ImageDataGenerator(
featurewise_center=datagen_featurewise_center,
samplewise_center=datagen_samplewise_center,
featurewise_std_normalization=datagen_featurewise_std_normalization,
samplewise_std_normalization=datagen_samplewise_std_normalization,
zca_whitening=datagen_zca_whitening,
zca_epsilon=datagen_zca_epsilon,
rotation_range=datagen_rotation_range,
width_shift_range=datagen_width_shift_range,
height_shift_range=datagen_height_shift_range,
brightness_range=datagen_brightness_range,
shear_range=datagen_shear_range,
zoom_range=datagen_zoom_range,
channel_shift_range=datagen_channel_shift_range,
fill_mode=datagen_fill_mode,
cval=datagen_cval,
horizontal_flip=datagen_horizontal_flip,
vertical_flip=datagen_vertical_flip,
rescale=datagen_rescale,
preprocessing_function=datagen_preprocessing_function,
data_format=datagen_data_format,
validation_split=datagen_validation_split,
dtype=datagen_datagen_dtype))
if not dataType == 'custom':
(x_train, y_train), _ = getattr(keras.datasets, dataType).load_data()
datagen.fit(x_train, y_train)
del x_train, y_train
elif dataType == 'standard':
datagen.mean = data_mu
datagen.std = data_sigma
return datagen
def fit_and_evaluate(X_train,y_train):
model = None
gc.collect()
model = createModel(X_train)
batch_size = 32
epochs = 30
gc.collect()
datagen = ImageDataGenerator(zoom_range = 0.2,horizontal_flip = False)
datagen.fit(X_train)
gc.collect()
train_x, val_x, train_y, val_y = train_test_split(X_train, y_train, test_size=0.1, random_state = np.random.randint(1,1000, 1)[0])
plotCategories(train_y,val_y)
results = model.fit_generator(datagen.flow(train_x,train_y,batch_size=batch_size), epochs = epochs,steps_per_epoch = X_train.shape[0] // batch_size ,callbacks=[early_stopping, model_checkpoint],
verbose=1,validation_data = (val_x,val_y))
gc.collect()
print("Val Score: ", model.evaluate(val_x, val_y))
return
def _datagen(self):
"""
Image Data Enhancement
"""
datagen = ImageDataGenerator(
rotation_range=10,
width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.05,
zoom_range=0.05,
horizontal_flip=True,
)
datagen.fit(self.DATASET.train_x)
return datagen.flow(
self.DATASET.train_x,
self.DATASET.train_y,
batch_size=self.BATCH_SIZE,
shuffle=True)
def augment_data(x_train, y_train, x_val, y_val, batch_size=128):
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=15,
horizontal_flip=True)
val_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=15,
horizontal_flip=True)
train_datagen.fit(x_train)
train_generator = train_datagen.flow(x_train,
y_train,
batch_size=batch_size)
val_datagen.fit(x_val)
val_generator = val_datagen.flow(x_val, y_val, batch_size=batch_size)
return train_generator, val_generator
def fit_and_evaluate(train_x, val_x, train_y, val_y, model, callbacks):
gc.collect()
batch_size = 32
epochs = 30
gc.collect()
datagen = ImageDataGenerator(zoom_range=0.2, horizontal_flip=False)
print("DataGen Started..")
datagen.fit(train_x)
print("DataGen Finished..")
gc.collect()
results = model.fit_generator(datagen.flow(train_x,
train_y,
batch_size=batch_size),
epochs=epochs,
callbacks=callbacks,
verbose=1,
validation_data=(val_x, val_y))
gc.collect()
print("Val Score: ", model.evaluate(val_x, val_y))
return model, results
def image_augmentation(img, label, augnum): #num means batch_size
#label = label.reshape((1,)+label.shape)
image_datagen = ImageDataGenerator(rotation_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
fill_mode='nearest')
label_datagen = ImageDataGenerator(rotation_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
fill_mode='nearest')
seed = random.randint(1, 10000)
n = 0
seed_ = 1
image_datagen.fit(img, seed=seed_)
label_datagen.fit(label, seed=seed_)
for batch in image_datagen.flow(img,
batch_size=1,
save_to_dir=IMG_PATH,
save_prefix='aug',
save_format='png',
seed=seed):
n += 1
if n > augnum:
break
n = 0
for batch in label_datagen.flow(label,
batch_size=1,
save_to_dir=LABEL_PATH,
save_prefix='aug_label',
save_format='png',
seed=seed):
n += 1
if n > augnum:
break
return
'''
class AugmentationGenerator(Sequence):
def __init__(self, X, y, batch_size, shuffle=True):
self.X = X
self.y = y
self.batch_size = batch_size
self.shuffle = shuffle
self.index = np.arange(len(X), dtype=int)
self.checked = []
self.augmenter = ImageDataGenerator(
featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False,
samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, rotation_range=0,
width_shift_range=0.1, height_shift_range=0.1, brightness_range=None, shear_range=0.0, zoom_range=0,
channel_shift_range=0., fill_mode='nearest', cval=0., horizontal_flip=True, vertical_flip=False,
rescale=None, preprocessing_function=None, data_format='channels_last', validation_split=0,
dtype='float32')
self.augmenter.fit(X)
self.on_epoch_end()
def __len__(self):
return int(np.ceil(len(self.X) / self.batch_size))
def __getitem__(self, index):
indexes = self.index[index * self.batch_size:(index + 1) * self.batch_size]
self.checked.extend(indexes)
return self.augmenter.flow(self.X[indexes], batch_size=len(indexes), shuffle=False).next(), self.y[indexes]
def on_epoch_end(self):
if len(np.unique(self.checked)) == len(self.X):
print("All checked")
self.checked = []
if self.shuffle:
np.random.shuffle(self.index)
def train_gen(self):
print(len(self.data_test))
datagen = ImageDataGenerator(rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1)
datagen.fit(self.data_train[0])
for i in range(len(self.models)):
self.models[i].fit_generator(
datagen.flow(self.data_train[0],
self.data_train[1],
batch_size=self.config.batch_size),
epochs=self.config.num_epochs,
steps_per_epoch=self.data_train[0].shape[0] //
self.config.batch_size,
validation_data=(self.data_test[0], self.data_test[1]),
callbacks=[
ReduceLROnPlateau(monitor='lr', patience=3, factor=0.1)
],
verbose=2)
self.models.append(self.models[i])
def train_model_with_generator(self):
self.logger.info('Training model with image data generator')
image_data_gen = ImageDataGenerator(rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=[0.8, 1.1],
brightness_range=[0.5, 1.5],
fill_mode='reflect')
image_data_gen.fit(self.trainX)
self.model.fit_generator(
image_data_gen.flow(self.trainX,
self.trainY,
batch_size=constants.BATCH_SIZE),
validation_data=(self.testX, self.testY),
steps_per_epoch=len(self.trainX) // constants.BATCH_SIZE,
epochs=constants.EPOCHS)
self.__evaluate_model()
def _train(self):
x_train, y_train = self.train_data
x_train, y_train = x_train[:NUM_SAMPLES], y_train[:NUM_SAMPLES]
x_test, y_test = self.test_data
x_test, y_test = x_test[:NUM_SAMPLES], y_test[:NUM_SAMPLES]
aug_gen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by dataset std
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# randomly rotate images in the range (degrees, 0 to 180)
rotation_range=0,
# randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
)
aug_gen.fit(x_train)
batch_size = self.config.get("batch_size", 64)
gen = aug_gen.flow(x_train, y_train, batch_size=batch_size)
self.model.fit_generator(
generator=gen,
epochs=self.config.get("epochs", 1),
validation_data=None)
# loss, accuracy
_, accuracy = self.model.evaluate(x_test, y_test, verbose=0)
return {"mean_accuracy": accuracy}
def _train(self):
x_train, y_train = self.train_data
x_test, y_test = self.test_data
aug_gen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by dataset std
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# randomly rotate images in the range (degrees, 0 to 180)
rotation_range=0,
# randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False,
)
aug_gen.fit(x_train)
gen = aug_gen.flow(
x_train, y_train, batch_size=self.config["batch_size"])
self.model.fit_generator(
generator=gen,
steps_per_epoch=50000 // self.config["batch_size"],
epochs=self.config["epochs"],
validation_data=None)
# loss, accuracy
_, accuracy = self.model.evaluate(x_test, y_test, verbose=0)
return {"mean_accuracy": accuracy}
def train(args, filepath, f_output, model_n, y_train, y_test, test_steps, x_test_dset=None, x_train=None,
method=None, save_all_weights=False):
print(model_n)
base_model_name = args.model_name
if args.extension is not None:
base_model_name = re.sub('_' + args.extension, '', base_model_name)
# Extracting statistics for every model-set combination and history for learning curves
history = []
test_acc = np.zeros(args.repetitions)
test_loss = np.zeros_like(test_acc)
training_time = []
inference_time = np.zeros_like(test_acc)
callbacks = []
n_classes = len(y_train[0])
y_test = np.argmax(y_test, axis=1)
agg_cm = np.zeros((n_classes, n_classes))
for i in range(args.repetitions):
if args.scheduler != 'NA':
sched = globals()[args.scheduler]
if 'stage' in args.scheduler:
print(args.scheduler)
cb_decayLR = tf.keras.callbacks.LearningRateScheduler(sched(args.learning_rate, args.num_epochs),
verbose=0)
else:
cb_decayLR = tf.keras.callbacks.LearningRateScheduler(sched, verbose=0)
else:
cb_decayLR = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1,
mode='auto', min_delta=0.0001, cooldown=0,
min_lr=args.learning_rate / 100)
if not callbacks:
callbacks.append(cb_decayLR)
else:
callbacks[0] = cb_decayLR
# Resetting the model for the next iteration
input_shape = x_train.shape[1:]
print('Loading model: ', base_model_name)
optimizer = tf.keras.optimizers.SGD(args.learning_rate, momentum=0.9, nesterov=True)
if method is not None:
print("Method: ", method)
model = select_model(input_shape, base_model_name, optimizer, args.weight_decay, method)
else:
model = select_model(input_shape, base_model_name, optimizer, args.weight_decay)
x_train, y_train = shuffle(x_train, y_train)
# Extract tranining and validation split indices
if args.val_split != 0:
train_ind, val_ind = validation_split(x_train, y_train, args.val_split, args.dataset == 'TOY')
# Timing training
start_train = time.time()
if args.augmentation:
datagen = ImageDataGenerator(
featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False,
samplewise_std_normalization=False, zca_whitening=False, zca_epsilon=1e-06, rotation_range=0,
width_shift_range=0.1, height_shift_range=0.1, brightness_range=None, shear_range=0.0, zoom_range=0,
channel_shift_range=0., fill_mode='nearest', cval=0., horizontal_flip=True, vertical_flip=False,
rescale=None, preprocessing_function=None, data_format='channels_last', validation_split=0,
dtype='float32')
datagen.fit(x_train[train_ind])
hist = model.fit_generator(datagen.flow(x_train[train_ind], y_train[train_ind], batch_size=args.batch_size),
epochs=args.num_epochs,
validation_data=(x_train[val_ind], y_train[val_ind]),
callbacks=callbacks, verbose=2)
else:
if args.val_split != 0:
hist = model.fit(x_train[train_ind], y=y_train[train_ind], batch_size=args.batch_size, epochs=args.num_epochs,
verbose=2, validation_data=(x_train[val_ind], y_train[val_ind]), callbacks=callbacks)
else:
hist = model.fit(x_train, y=y_train, batch_size=args.batch_size,
epochs=args.num_epochs,
verbose=2, validation_data=x_test_dset, validation_steps=test_steps,
callbacks=callbacks)
training_time.append(time.time() - start_train)
history.append(hist.history)
test_loss[i], test_acc[i] = model.evaluate(x_test_dset, steps=test_steps, verbose=0)
start_inference = time.time()
y_pred = model.predict(x_test_dset, steps=test_steps, verbose=0)
inference_time[i] = time.time() - start_inference
# From one-hot to single class prediction
y_pred = np.argmax(y_pred, axis=1)
agg_cm += confusion_matrix(y_test, y_pred)
if save_all_weights:
model.save(filepath['models'] + filepath['dataset'] + model_n + '_it' + str(i) + '.h5')
# Checkpoint model in last iter
else:
if i == args.repetitions - 1:
model.save(filepath['models'] + filepath['dataset'] + model_n + '.h5')
# Store history
with open(filepath['history'] + filepath['dataset'] + 'history_' + model_n + '.txt', 'wb') as f_history:
pickle.dump(history, f_history)
# Extract and output metrics
mean_test_loss = np.mean(test_loss)
std_test_loss = np.std(test_loss, ddof=1)
mean_test_acc = np.mean(test_acc)
std_test_acc = np.std(test_acc, ddof=1)
mean_inference_time = np.mean(inference_time)
std_inference_time = np.std(inference_time, ddof=1)
agg_cm /= args.repetitions
if True:
agg_cm = np.round(agg_cm/np.sum(agg_cm, axis=1), 3)
# Writing statistics to file
print("****************************************", file=f_output)
print("Model: ", model_n, file=f_output)
print(f"Mean test loss: {mean_test_loss} +- {std_test_loss}", file=f_output)
print(f"Mean test accuracy: {mean_test_acc} +- {std_test_acc}\n", file=f_output)
print("Aggregated confusion matrix:", file=f_output)
print(f"{agg_cm}\n", file=f_output)
print(f"Mean training time: {np.mean(training_time)} +- {np.std(training_time, ddof=1)}", file=f_output)
print(f"Mean inference time: {mean_inference_time} +- {std_inference_time}", file=f_output)
print("****************************************\n\n\n", file=f_output)
learning_curves(history, model_n=model_n, filepath=filepath['graphs'] + filepath['dataset'])
def start_training(tn,vn,ims,bas,epc):
training_num = tn
validation_num = vn
image_size = ims
batch_size = bas
epochs = epc
WEIGHTS_FOLDER = './weights/'
#WEIGHTS_FOLDER = save_path
if not os.path.exists(WEIGHTS_FOLDER):
os.mkdir(WEIGHTS_FOLDER)
# Check if image dimension is correct.
if type(image_size) is list:
val1 = image_size[0]
val2 = image_size[1]
if val1 < 139 or val2 < 139:
print("The size is not ok....")
sys.exit(2)
elif type(image_size) is int:
if image_size <139:
print("The size is not ok...")
sys.exit(2)
# Show the training condition
print("Image size is {}".format(image_size))
print("The batch_size is {}".format(batch_size))
print("The epochs is {}".format(epochs))
# Load images and data from cifar 10
(x_train,y_train),(x_validation,y_validation) = cifar10.load_data()
# Load part of train and test data.
x_train = x_train[:training_num]
x_validation = x_validation[:validation_num]
Y_train = y_train[:training_num]
Y_validation = y_validation[:validation_num]
print("Total Train & Validation Num as shown below")
print("Num of training images : {}".format(x_train.shape[0]))
print("Num of validation images : {}".format(x_validation.shape[0]))
X_train,X_validation = helpResize(x_train,x_validation,image_size)
# Check if both of the list has the correct length.
Y_new_train = np.array([np.zeros(10) for x in range(len(Y_train))],dtype='float32')
for i,x in enumerate(Y_train):
Y_new_train[i][x] = 1
Y_new_val = np.array([np.zeros(10) for x in range(len(Y_validation))],dtype='float32')
for i,x in enumerate(Y_validation):
Y_new_val[i][x] = 1
# This could also be the output of a different Keras model or layer
if type(image_size) is list:
input_shape = tuple(image_size) + (3,)
else:
input_shape = (image_size,image_size,3)
base_model = InceptionV3(weights='imagenet', include_top=False,input_shape=input_shape)
# Get the output of the Inception V3 pretrain model.
x = base_model.output
# Works same as Flatten(), but Flatten has larger dense layers, it might cause worse overfitting.
# However, if the user has a larger dataset then the user can use Flatten() instead of GlobalAveragePooling2D or GlobalMaxPooling2D
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)
# Use SGD as an optimizer
model.compile(optimizer=SGD(lr=0.0001, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
datagen = ImageDataGenerator(
rotation_range=0, # Randomly rotate images in the range (degrees, 0 to 180)
# Randomly shift images horizontally (fraction of total width)
width_shift_range=0.1,
# Randomly shift images vertically (fraction of total height)
height_shift_range=0.1,
zoom_range=0., # set range for random zoom
# Set the mode for filling points outside the input boundaries
horizontal_flip=True, # randomly flip images
)
datagen.fit(X_train)
histories = NCHC_CallBack()
c_time = "{}_{}_{}_{}_{}".format(time.localtime().tm_year,time.localtime().tm_mon,time.localtime().tm_mday,time.localtime().tm_hour,time.localtime().tm_min)
mc = ModelCheckpoint(WEIGHTS_FOLDER+c_time+"_weights.{epoch:02d}-acc-{acc:.2f}-loss-{loss:.2f}.hdf5",
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto', period=1)
# Fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(X_train,
Y_new_train,
batch_size=batch_size),
callbacks = [ histories,mc ], #added here
epochs=epochs,
validation_data=(X_validation, Y_new_val)
)
K.clear_session()
del model
def evaluate_on_cifar10():
total_depth = 100
n_blocks = 3
depth = (total_depth - 4) // n_blocks
growth_rate = 12
filters = growth_rate * 2
# region Model
input_layer = Input(shape=[32, 32, 3])
layer = input_layer
layer = Conv2D(filters=filters, kernel_size=3, strides=1,
padding="same")(layer)
for k in range(n_blocks):
layer = DenseBlock2D(kernel_size=3,
growth_rate=growth_rate,
depth=depth,
use_batch_normalization=True)(layer)
if k < (n_blocks - 1):
filters += growth_rate * depth // 4
layer = transition_block(layer, filters)
else:
layer = AveragePooling2D(pool_size=8)(layer)
layer = Flatten()(layer)
layer = Dense(units=10, activation="softmax")(layer)
model = Model(inputs=input_layer, outputs=layer)
model.summary()
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["acc"])
# endregion
# region Data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype(np.float32) / 255.0
x_test = x_test.astype(np.float32) / 255.0
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
generator = ImageDataGenerator(rotation_range=15,
width_shift_range=5. / 32,
height_shift_range=5. / 32,
horizontal_flip=True)
generator.fit(x_train, seed=0)
# endregion
log_dir = "../logs/tests/dense_block_cifar10/{}".format(int(time()))
log_dir = os.path.normpath(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, profile_batch=0)
model.fit_generator(generator.flow(x_train, y_train, batch_size=64),
steps_per_epoch=100,
epochs=300,
validation_data=(x_test, y_test),
validation_steps=100,
verbose=1,
callbacks=[tensorboard])
my_new_model = Sequential()
my_new_model.add(
ResNet50(include_top=False, pooling='avg', weights=resnet_weights_path))
my_new_model.add(Dense(num_classes, activation='softmax'))
# Say not to train first layer (ResNet) model. It is already trained
my_new_model.layers[0].trainable = False
my_new_model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
train_X, test_X, train_y, test_y = train_test_split(x, y, test_size=0.25)
datagen = ImageDataGenerator(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(train_X)
# fits the model on batches with real-time data augmentation:
my_new_model.fit_generator(
datagen.flow(train_X, train_y, batch_size=32),
steps_per_epoch=3,
epochs=2,
)
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint
from tensorflow.keras.callbacks import ModelCheckpoint
# 透過 data augmentation 產生訓練與驗證用的影像資料
train_datagen = ImageDataGenerator(rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
channel_shift_range=10,
horizontal_flip=True,
fill_mode='nearest')
train_datagen.fit(x_train)
# 以訓練好的 ResNet50 為基礎來建立模型，
# 捨棄 ResNet50 頂層的 fully connected layers
net = ResNet50(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
x = net.output
x = Flatten()(x)
# 增加 DropOut layer
x = Dropout(0.5)(x)
# 增加 Dense layer，以 softmax 產生個類別的機率值
output_layer = Dense(NUM_CLASSES, activation='softmax', name='softmax')(x)
def main(_):
# get current time
t = tuple(localtime(time()))[1:5]
# load FASHION-MNIST dataset
data = input_data.read_data_sets('data/fashion', one_hot=True)
# get training, validation, test data and reshape data
train_images, train_labels = reshape_data(data.train)
validation_images, validation_labels = reshape_data(data.validation)
test_images, test_labels = reshape_data(data.test)
# training data generator with data augmentation
train_datagen = ImageDataGenerator(rotation_range=30,
horizontal_flip=True,
vertical_flip=True)
train_datagen.fit(train_images)
# Shapes of training dataset
print("Training set images shape: {shape}".format(shape=data.train.images.shape))
print("Training set labels shape: {shape}".format(shape=data.train.labels.shape))
# Shapes of validation dataset
print("Validation set images shape: {shape}".format(shape=data.validation.images.shape))
print("Validation set labels shape: {shape}".format(shape=data.validation.labels.shape))
# Shapes of test dataset
print("Test set images shape: {shape}".format(shape=data.test.images.shape))
print("Test set labels shape: {shape}".format(shape=data.test.labels.shape))
# summary directory path to save summaries
summary_dir = '/tmp/FASHION-MNIST/run-%02d%02d-%02d%02d' % t
# checkpoint directory path
ckpt_dir = 'checkpoint'
x = tf.placeholder(tf.float32, [None, 1, 28, 28])
y = tf.placeholder(tf.float32, [None, 10])
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(FLAGS.lr, global_step, 300, 0.95, staircase=True)
with tf.name_scope('logits'):
logits, is_training = build_graph(x, FLAGS.dropout_rate)
prediction = tf.nn.softmax(logits)
with tf.name_scope('loss'):
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
tf.summary.scalar('cross_entropy', loss_op)
with tf.name_scope('optimizer'):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.minimize(loss_op, global_step)
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
init = tf.global_variables_initializer()
# model saver
saver = tf.train.Saver()
# Start training
with tf.Session() as sess:
# training and validation dataset summary writer
train_writer = tf.summary.FileWriter(summary_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(summary_dir + '/validation', sess.graph)
# Run the initializer
sess.run(init)
max_validation_acc = 0.9
for step, (batch_x, batch_y) in enumerate(train_datagen.flow(train_images,
train_labels,
batch_size=FLAGS.batch_size)):
_, summary = sess.run([train_op, merged], feed_dict={x: batch_x,
y: batch_y,
is_training: True})
train_writer.add_summary(summary, step)
if step % 50 == 0:
# calculate batch loss and accuracy
loss, acc, summary = sess.run([loss_op, accuracy, merged], feed_dict={x: validation_images,
y: validation_labels,
is_training: False})
validation_writer.add_summary(summary, step)
print("Step {0}, Minibatch Loss= {1}, Validation Accuracy= {2}".format(str(step), "{:.4f}".format(loss),
"{:.3f}".format(acc)))
if acc > max_validation_acc:
# save model
saver.save(sess, os.path.join(ckpt_dir, 'ckpt'), global_step=step)
max_validation_acc = acc
if step >= FLAGS.num_steps:
break
print("Optimization Finished!")
# restore best model
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if ckpt and ckpt.model_checkpoint_path:
ckpt_name = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_name)
print('Restore {}'.format(ckpt_name))
# calculate test accuracy
test_accuracy = sess.run(accuracy, feed_dict={x: test_images,
y: test_labels,
is_training: False})
print("Test Accuracy: {}".format(test_accuracy))
x_test = (x_test - mean) / (std + 1e-7)
y_train = np_utils.to_categorical(y_train, num_classes)
y_test = np_utils.to_categorical(y_test, num_classes)
################### Augmentation #####################
if not data_augmentation:
print('')
print('>>>>>>>Not using data augmentation <<<<<<<<')
datagen = ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False,
horizontal_flip=False)
datagen.fit(x_train)
else:
print('')
print('>>>>>>>> Using real-time data augmentation <<<<<<<<<<<')
'''
datagen = ImageDataGenerator(featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=0,
width_shift_range=0,
height_shift_range=0,
brightness_range=(-0, 0),
zoom_range=(-0, 0),
horizontal_flip=False)
"""Training the Model"""
print("reading data and preproccessing")
X_train, Y_train = create_training_data()
print ('reading and preproccessing finished')
X_train, X_val, Y_train, Y_val = split_data_to_train_and_validation(X_train, Y_train, 0.1)
print('starting data augmentation')
data_generator_with_aug = ImageDataGenerator(
horizontal_flip=False,
vertical_flip=False,
width_shift_range = 0.1,
height_shift_range = 0.1,
rotation_range=5
)
data_generator_with_aug.fit(X_train)
cnn_model = build_model(save=True)
History = fit_model(cnn_model, data_generator_with_aug.flow(X_train,Y_train, batch_size=64), (X_val,Y_val), 5, save=True )
"""Read and create Test data along with its labels"""
def create_testing_data():
X_test = []
Y_test = []
for test in TEST_DATA:
data = test.split("_")
category_number = CATEGORIES.index(data[0])
try:
img_array = cv2.imread(os.path.join(TEST_DIR,test), cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array, (IMAGE_SIZE, IMAGE_SIZE))
def main():
tf.random.set_seed(42)
block_count = 4
basic_block_count = 8
input_shape = (32, 32, 3)
layers_params = {
"rank": 2,
"head_size": 8,
"head_count": 8,
"basic_block_count": basic_block_count,
"kernel_size": 3,
"strides": 1,
"dilation_rate": 1,
"activation": "relu",
"kernel_regularizer": None,
"bias_regularizer": None,
"activity_regularizer": None,
"kernel_constraint": None,
"bias_constraint": None,
}
layers = [
ResBlock2D(filters=16,
basic_block_count=basic_block_count,
kernel_size=7,
input_shape=input_shape),
MaxPooling2D(4)
]
for i in range(1, block_count):
layer = ResSASABlock(**layers_params)
layers.append(layer)
layers_params["head_size"] *= 2
layers.append(MaxPooling2D(2))
layers.append(Flatten())
layers.append(
Dense(units=10,
activation="softmax",
kernel_initializer=VarianceScaling()))
model = Sequential(layers=layers,
name="StandAloneSelfAttentionBasedClassifier")
model.summary()
model.compile("adam", loss="categorical_crossentropy", metrics=["acc"])
# region Data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype(np.float32) / 255.0
x_test = x_test.astype(np.float32) / 255.0
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
generator = ImageDataGenerator(rotation_range=15,
width_shift_range=5. / 32,
height_shift_range=5. / 32,
horizontal_flip=True)
generator.fit(x_train)
# endregion
log_dir = "../../logs/tests/stand_alone_self_attention_cifar10/{}".format(
int(time()))
log_dir = os.path.normpath(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, profile_batch="500,520")
model.fit(generator.flow(x_train, y_train, batch_size=64),
steps_per_epoch=100,
epochs=300,
validation_data=(x_test, y_test),
validation_steps=100,
verbose=1,
callbacks=[tensorboard])
