def main():
now = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
model_name = 'simpleCNN_' + now + '.h5'
batch_size = 256
num_epochs = 30
lr = .001
num_train_samples = len(os.listdir('./data/train/cancer')) + len(os.listdir('./data/train/healthy'))
num_valid_samples = len(os.listdir('./data/validation/cancer')) + len(os.listdir('./data/validation/healthy'))
# Build our model
input_tensor = Input(shape=(96, 96, 3))
x = layers.Conv2D(32, (3, 3))(input_tensor)
x = layers.MaxPooling2D((2, 2))(x)
x = layers.Conv2D(64, (3, 3))(x)
x = layers.MaxPooling2D((2, 2))(x)
x = layers.Conv2D(128, (3, 3))(x)
x = layers.MaxPooling2D((2, 2))(x)
x = layers.Conv2D(128, (3, 3))(x)
x = layers.MaxPooling2D((2, 2))(x)
x = layers.Flatten()(x)
x = layers.Dropout(.5)(x)
x = layers.Dense(512, activation='relu')(x)
output_tensor = layers.Dense(1, activation='sigmoid')(x)
model = Model(input_tensor, output_tensor)
model.summary()
# Get things ready to train: should adjust learning rate, etc.
model.compile(optimizer=Adam(lr), loss='binary_crossentropy', metrics=['acc'])
train_generator = train_gen(batch_size)
validation_generator = valid_gen(batch_size)
steps_per_epoch = num_train_samples / batch_size
validation_steps = num_valid_samples / batch_size
# Basic callbacks
checkpoint = callbacks.ModelCheckpoint(filepath='./models/' + model_name,
monitor='val_loss',
save_best_only=True)
early_stop = callbacks.EarlyStopping(monitor='val_acc',
patience=3)
csv_logger = callbacks.CSVLogger('./logs/' + model_name.split('.')[0] + '.csv')
callback_list = [checkpoint, early_stop, csv_logger]
# Training begins
history = model.fit_generator(train_generator,
steps_per_epoch=steps_per_epoch,
epochs=num_epochs,
verbose=1,
callbacks=callback_list,
validation_data=validation_generator,
validation_steps=validation_steps)
model.save('./models/' + model_name)
make_plots(history, model_name)
def train(save_dir, batch_size, lr, shift_fraction, epochs, model, data, running_time):
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
class_weights_array = class_weight.compute_class_weight(
#None
'balanced'
,np.unique(np.argmax(y_train, axis=1))
,np.argmax(y_train, axis=1))
class_weights={0:class_weights_array[1],1:class_weights_array[0]}
# callbacks
log_dir = save_dir + '\\tensorboard-logs-dd' + '\\' + running_time
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log = callbacks.CSVLogger(save_dir + '\\log-dd.csv')
tb = callbacks.TensorBoard(log_dir=log_dir,
batch_size=batch_size)
checkpoint = callbacks.ModelCheckpoint(save_dir + '\\weights-dd-{epoch:02d}.h5', monitor='val_acc',
save_best_only=True, save_weights_only=True, verbose=1)
# compile the model
model.compile(optimizer=optimizers.Adam(lr=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
# Begin: Training with data augmentation---------------------------------------------------------------------#
def train_generator(x, y, batch_size, savedir, shift_fraction=0.):
if not os.path.exists(savedir):
os.makedirs(savedir)
train_datagen = ImageDataGenerator(width_shift_range=shift_fraction,
samplewise_std_normalization = False,
height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST
generator = train_datagen.flow(x, y, batch_size=batch_size, shuffle=False)
while 1:
x_batch, y_batch = generator.next()
yield (x_batch, y_batch)
# Training with data augmentation. If shift_fraction=0., also no augmentation.
print(class_weights)
model.fit(x_train, y_train, batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
callbacks=[log, tb, checkpoint],
class_weight= class_weights,
shuffle=True)
# End: Training with data augmentation -----------------------------------------------------------------------#
model.save_weights(save_dir + 'trained_model_dd_toxo.h5')
print('Trained model saved to \'%s \\trained_mode_dd_toxo.h5\'' % save_dir)
plot_log(os.path.join(save_dir, 'log-dd.csv'), show=True)
return model
def get_callbacks(self): checkpointer = callbacks.ModelCheckpoint(filepath=self.save_path, monitor='val_loss', verbose=1, save_best_only=True) early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-7, patience=15, verbose=1, mode='auto') reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1) csv_logger = callbacks.CSVLogger(self.train_log_path, separator=',', append=False) # tensorboard = TensorBoard(log_dir='logs/', histogram_freq=0, batch_size=self.batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) return [early_stopping,checkpointer,reduce_lr,csv_logger]
def pretrain(self,
x,
y=None,
optimizer='adam',
epochs=200,
batch_size=256,
save_dir='results/temp'):
print('...Pretraining...')
self.autoencoder.compile(optimizer=optimizer, loss='mse')
csv_logger = callbacks.CSVLogger(save_dir + '/pretrain_log.csv')
cb = [csv_logger]
if y is not None:
class PrintACC(callbacks.Callback):
def __init__(self, x, y):
self.x = x
self.y = y
super(PrintACC, self).__init__()
def on_epoch_end(
self,
epoch,
logs=None): #called at the end of every epoch?
if int(epochs / 10) != 0 and epoch % int(
epochs /
10) != 0: # 只在epochs的10%的迭代次数之内运行以下的print 代码
return
feature_model = Model(
self.model.input,
self.model.get_layer(
'encoder_%d' %
(int(len(self.model.layers) / 2) - 1)).output)
features = feature_model.predict(
self.x
) #pretrain训练的是自编码器部分，这里是encoder的输出，在embedding后的向量空间上进行KMEANS可以观察encoding的效果？
km = KMeans(n_clusters=len(np.unique(self.y)),
n_init=20,
n_jobs=4)
y_pred = km.fit_predict(features)
# print()
print(' ' * 8 + '|==> acc: %.4f, nmi: %.4f <==|' %
(metrics.acc(self.y, y_pred),
metrics.nmi(self.y, y_pred)))
cb.append(PrintACC(x, y))
# begin pretraining
t0 = time()
self.autoencoder.fit(x,
x,
batch_size=batch_size,
epochs=epochs,
callbacks=cb)
print('Pretraining time: %ds' % round(time() - t0))
self.autoencoder.save_weights(save_dir + '/ae_weights.h5')
print('Pretrained weights are saved to %s/ae_weights.h5' % save_dir)
self.pretrained = True
def train_top_model(y_train, nb_class, max_index, epochs, batch_size,
input_folder, result_path):
#load bottleneck predictions
train_data = np.load(result_path + 'bottleneck_features_train.npy')
train_labels = y_train
print(train_data.shape, train_labels.shape)
#make top model
#final output must be only 1 node, as we are only regressing one value
model = Sequential()
model.add(Flatten(input_shape=train_data.shape[1:]))
model.add(Dropout(0.3))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(nb_class, activation='relu'))
model.add(Dense(1, activation=None))
#set optimizer settings and compile model
lr = 0.05
decay = 5e-5
momentum = 0.75
optimizer = optimizers.SGD(lr=lr,
decay=decay,
momentum=momentum,
nesterov=True)
loss = 'mse'
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
bottleneck_log = result_path + 'training_' + str(
max_index) + '_bnfeature_log.csv'
csv_logger_bnfeature = callbacks.CSVLogger(bottleneck_log)
earlystop = EarlyStopping(monitor='val_acc',
min_delta=0.0001,
patience=3,
verbose=1,
mode='auto')
#train top model on bottleneck features
model.fit(train_data,
train_labels,
epochs=epochs,
batch_size=batch_size,
shuffle=True,
callbacks=[csv_logger_bnfeature, earlystop],
verbose=2,
validation_split=0.2)
with open(bottleneck_log, 'a') as log:
log.write('\n')
log.write('input images: ' + input_folder + '\n')
log.write('batch_size:' + str(batch_size) + '\n')
log.write('learning rate: ' + str(lr) + '\n')
log.write('learning rate decay: ' + str(decay) + '\n')
log.write('momentum: ' + str(momentum) + '\n')
log.write('loss: ' + loss + '\n')
#save top model weights
model.save_weights(result_path + 'bottleneck_fc_model.h5')
def pretrain(self,
x,
y=None,
optimizer='adam',
epochs=200,
batch_size=256,
save_dir='results/temp'):
print('...Pretraining...')
self.autoencoder.compile(optimizer=optimizer, loss='mse')
csv_logger = callbacks.CSVLogger(save_dir + '/pretrain_log.csv')
cb = [csv_logger]
if y is not None:
class PrintACC(callbacks.Callback):
def __init__(self, x, y):
self.x = x
self.y = y
super(PrintACC, self).__init__()
def on_epoch_end(self, epoch, logs=None):
if int(epochs / 10) != 0 and epoch % int(epochs / 10) != 0:
return
feature_model = Model(
self.model.input,
self.model.get_layer(
'encoder_%d' %
(int(len(self.model.layers) / 2) - 1)).output)
features = feature_model.predict(self.x)
km = KMeans(n_clusters=len(np.unique(self.y)),
n_init=20,
n_jobs=4)
y_pred = km.fit_predict(features)
# print()
print(' ' * 8 + '|==> acc: %.4f, nmi: %.4f <==|' %
(metrics.acc(self.y, y_pred),
metrics.nmi(self.y, y_pred)))
cb.append(PrintACC(x, y))
# begin pretraining
t0 = time()
temp = []
for i in range(x.shape[0]):
t = []
for k in range(self.dims[-1]):
t.append(0)
temp.append(t)
temp = np.array(temp)
self.autoencoder.fit([x, temp],
x,
batch_size=batch_size,
epochs=epochs,
callbacks=cb)
print('Pretraining time: %ds' % round(time() - t0))
self.autoencoder.save_weights(save_dir + '/ae_weights.h5')
print('Pretrained weights are saved to %s/ae_weights.h5' % save_dir)
self.pretrained = True
def fit(self,
exprs=None,
classes=None,
model_dir=None,
model_name='my_rlvae'):
"""
Function to fit the complete resVAE model to the provided training data.
:param exprs: Input matrix (samples x features)
:param classes: One-hot encoded or partial class identity matrix (samples x classes)
:param model_dir: Directory to save training logs in
:param model_name: Name of the model (for log files
:return: Returns a keras history object and a dictionary with scores
"""
assert self.compiled, print(
'Please compile the model first by running rlvae.compile()')
batch_size = self.config['BATCH_SIZE']
epochs = self.config['EPOCHS']
steps_per_epoch = self.config['STEPS_PER_EPOCH']
validation_split = self.config['VALIDATION_SPLIT']
callback = []
if self.config['CB_LR_USE']:
callback.append(
callbacks.ReduceLROnPlateau(
monitor=self.config['CB_MONITOR'],
factor=self.config['CB_LR_FACTOR'],
patience=self.config['CB_LR_PATIENCE'],
min_delta=self.config['CB_LR_MIN_DELTA'],
verbose=True))
if self.config['CB_ES_USE']:
callback.append(
callbacks.EarlyStopping(
monitor=self.config['CB_MONITOR'],
patience=self.config['CB_ES_PATIENCE'],
min_delta=self.config['CB_ES_MIN_DELTA'],
verbose=True))
if model_dir is not None:
callback.append(
callbacks.CSVLogger(
os.path.join(model_dir, str(model_name + '.log'))))
decoder_regularizer = self.config['DECODER_REGULARIZER']
decoder_regularizer_initial = self.config[
'DECODER_REGULARIZER_INITIAL']
if decoder_regularizer in ['var_l1', 'var_l2', 'var_l1_l2']:
callback.append(
callbacks.
LambdaCallback(on_epoch_end=lambda epoch, logs: K.set_value(
self.l_rate, decoder_regularizer_initial * (epoch + 1))))
history = self.resvae_model.fit([exprs, classes],
batch_size=batch_size,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
validation_split=validation_split,
callbacks=callback)
self.isfit = True
fpc_real = self.calc_fpc()
scores = {'fpc_real': fpc_real}
return history, scores
def train(model, data, args, dirs):
"""
The function which defines the training loop of the model
Parameters
----------
model : `keras.models.Model`
The structure of the model which is to be trained
data : `tuple`
The training and validation data
args : `dict`
The argument dictionary which defines other parameters at training time
dirs : `string`
Filepath to store the logs
"""
# Extract the data
(x_train, y_train), (x_val, y_val) = data
# callbacks
log = callbacks.CSVLogger(dirs + '/log.csv')
tb = callbacks.TensorBoard(log_dir=dirs + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=int(args.debug))
checkpoint = callbacks.ModelCheckpoint(dirs + '/model.h5',
monitor='val_acc',
save_best_only=True,
save_weights_only=False,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (args.lr_decay**epoch))
# compile the model
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss='binary_crossentropy',
metrics=['acc'])
# Training without data augmentation:
model.fit(x_train,
y_train,
batch_size=args.batch_size,
epochs=args.epochs,
verbose=1,
validation_data=(x_val, y_val),
callbacks=[
log, tb, checkpoint, lr_decay
]) #, roc_auc_callback((x_train, y_train), (x_val, y_val))])
# Save the trained model
model.save(dirs + '/trained_model.h5')
# Plot the training results
plot_log(dirs, show=False)
return model
def train(model, train, dev, test, save_directory, optimizer, epoch, batch_size, schedule):
(X_train, Y_train) = train
(X_dev, Y_dev) = dev
(X_test, Y_test) = test
# Callbacks
log = callbacks.CSVLogger(filename=save_directory + '/log.csv')
tb = callbacks.TensorBoard(log_dir=save_directory + '/tensorboard-logs', batch_size=batch_size)
checkpoint = callbacks.ModelCheckpoint(filepath=save_directory + '/weights-improvement-{epoch:02d}.hdf5',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(schedule=schedule, verbose=1)
# compile the model
model.compile(optimizer=optimizer,
loss=[margin_loss],
metrics=['accuracy'])
history = model.fit(x=X_train,
y=Y_train,
validation_data=[X_dev, Y_dev],
batch_size=batch_size,
epochs=epoch,
callbacks=[log, tb, checkpoint, lr_decay],
shuffle=True,
verbose=1)
score = model.evaluate(X_test, Y_test, batch_size=batch_size)
print colored(score, 'green')
print(history.history.keys())
# Summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['training accuracy', 'testing accuracy'], loc='upper left')
plt.savefig(save_directory + '/model_accuracy.png')
plt.close()
# Summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['training loss', 'testing loss'], loc='upper left')
plt.savefig(save_directory + '/model_loss.png')
plt.close()
model.save_weights(save_directory + '/trained_model.h5')
def train_model(model, args):
print('Loading train data!')
images_train, images_mask_train = load_data()
# callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
# 查看tensorboard:./ python -m tensorboard.main --logdir=./
tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=args.debug)
checkpoint = callbacks.ModelCheckpoint(
args.save_dir + '/trained_model.h5',
monitor='val_loss',
save_best_only=True,
save_weights_only=True,
verbose=1,
mode='min',
)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (0.99**epoch))
early_stopping = keras.callbacks.EarlyStopping(
monitor='val_loss',
patience=args.patience,
verbose=0,
mode='min',
)
# 断点续存
# model = keras.models.load_model("trained_model-old.h5",custom_objects={'bce_dice_loss': bce_dice_loss,
# 'mean_iou': mean_iou})
# compile the model
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss=bce_dice_loss,
metrics=["accuracy", mean_iou])
# Fitting model
model.fit(images_train,
images_mask_train,
batch_size=args.batch_size,
nb_epoch=args.epochs,
verbose=1,
shuffle=True,
validation_split=0.2,
callbacks=[log, tb, checkpoint, lr_decay, early_stopping])
# save model
model.save_weights(args.save_dir + '/trained_model.h5')
print('Trained model saved to \'%s/trained_model.h5\'' % args.save_dir)
plot_log(args.save_dir + '/log.csv', show=True)
return model
def setup_callbacks(filename_head):
""" Set up checkpoint callback """
checkpoint_filename = os.path.join(CHECKPOINT_DIR, filename_head + '_{epoch:02d}_chkpt.hdf5')
checkpoint_callback = callbacks.ModelCheckpoint(checkpoint_filename)
log_filename = os.path.join(LOGS_DIR, filename_head + '.log.csv')
csv_logger_callback = callbacks.CSVLogger(log_filename, separator=',', append=False)
return [checkpoint_callback, csv_logger_callback]
def loaddata_andtrain(dataframeip,batchsize,split_number,epnumber,routingnumber,initilr,lrdecay,x_train,y_train,x_test,y_test,recover=False):
save_dir='/home/ubuntu/Projects/tnc_ai/peter/model/result'
(x_train,y_train), (x_test,y_test)=(x_train,y_train),(x_test,y_test)
df=pd.read_csv('/home/ubuntu/Projects/tnc_ai/peter/model/EO_rank_data.csv')
model= CapsNet(input_shape=x_train.shape[1:],n_class=len(np.unique(df['specie']))-1,routings=routingnumber)
lr_decay=callbacks.LearningRateScheduler(schedule=lambda epoch: initilr*(lrdecay**epoch))
log = callbacks.CSVLogger(save_dir + '/log.csv')
checkpoint = callbacks.ModelCheckpoint(save_dir + '/weights-{epoch:02d}.h5', monitor='val_capsnet_acc',
save_best_only=True, save_weights_only=True, verbose=1)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=[categorical_accuracy])
# if recover:
# from glob import glob
# weights = glob(os.path.join(save_dir, '*.h5'))
# weights = sorted(weights, key=lambda x: x.split('-')[-1].split('.'))
# model.load_weights(weights[-1])
train_data_gen_args = dict(featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=90,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2)
def train_generator(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(**train_data_gen_args)
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
yield ([x_batch, y_batch], [y_batch, x_batch])
model.fit_generator(generator=train_generator(x_train,y_train,batchsize),
steps_per_epoch=int(y_train.shape[0] / batchsize),
epochs=epnumber,
validation_data=[[x_test, y_test], [y_test, x_test]],
callbacks=[lr_decay, log, checkpoint]
)
return model
def train(model, data, args):
"""
Training a CapsuleNet
:param model: the CapsuleNet model
:param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))`
:param args: arguments
:return: The trained model
"""
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
# callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=int(args.debug))
checkpoint = callbacks.ModelCheckpoint(args.save_dir +
'/weights-{epoch:02d}.h5',
monitor='val_capsnet_acc',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (args.lr_decay**epoch))
# compile the model
model.compile(optimizer=optimizers.SGD(lr=args.lr),
loss=[margin_loss, 'mse'],
loss_weights=[1., args.lam_recon],
metrics={'capsnet': 'accuracy'})
# Begin: Training with data augmentation ---------------------------------------------------------------------#
def train_generator(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(
width_shift_range=shift_fraction,
height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
yield ([x_batch, y_batch], [y_batch, x_batch])
# Training with data augmentation. If shift_fraction=0., also no augmentation.
model.fit_generator(
generator=train_generator(x_train, y_train, args.batch_size,
args.shift_fraction),
steps_per_epoch=int(y_train.shape[0] / args.batch_size),
epochs=args.epochs,
validation_data=[[x_test, y_test], [y_test, x_test]],
callbacks=[log, tb, checkpoint, lr_decay])
model.save_weights(args.save_dir + '/trained_model.h5')
print('Trained model saved to \'%s/trained_model.h5\'' % args.save_dir)
from utils import plot_log
plot_log(args.save_dir + '/log.csv', show=True)
return model
def train(self,
model_file,
csv_log_file,
load_weights=True,
save_weights=True):
model_path = os.path.split(model_file)[0]
if not os.path.exists(model_path):
os.makedirs(model_path)
# 载入模型
if os.path.exists(model_file) and load_weights:
self.sketch_model.load_weights(model_file, skip_mismatch=True)
# Callback
tensor_board = callbacks.TensorBoard(log_dir=self.summary_path,
histogram_freq=0,
update_freq=1000)
test_callback = callbacks.LambdaCallback(
on_epoch_end=lambda epoch, logs: self._test(epoch + 1))
csv_logger = callbacks.CSVLogger(csv_log_file)
# 测试
self._test("first")
# 训练：输入和输出在Model中定义
# 没有数据增强
# self.sketch_model.fit(x={'inputs': self.sketch_train_image}, y=[self.sketch_train_label],
# batch_size=self.batch_size, epochs=self.max_epoch,
# verbose=2, callbacks=[tensor_board, csv_logger, test_callback])
# 数据增强
data_gen = preprocessing.image.ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
rotation_range=30,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2)
data_gen.fit(self.sketch_train_image)
self.sketch_model.fit_generator(
generator=data_gen.flow(self.sketch_train_image,
self.sketch_train_label,
batch_size=self.batch_size),
epochs=self.max_epoch,
verbose=2,
callbacks=[tensor_board, csv_logger, test_callback],
steps_per_epoch=len(self.sketch_train_image) // self.batch_size)
# 测试
self._test("final")
# 保存模型
if save_weights:
self.sketch_model.save_weights(model_file)
pass
def pretrain(self,
x,
y=None,
optimizer='adam',
epochs=200,
batch_size=256,
save_dir='results/temp'):
print('...Pretraining...')
self.autoencoder.compile(optimizer=optimizer, loss='mse')
csv_logger = callbacks.CSVLogger(save_dir + '/pretrain_log.csv')
cb = [csv_logger]
if y is not None:
class PrintACC(callbacks.Callback):
def __init__(self, x, y):
self.x = x
self.y = y
super(PrintACC, self).__init__()
def on_epoch_end(self, epoch, logs=None):
if epoch % int(epochs / 10) != 0:
return
feature_model = Model(
self.model.input,
self.model.get_layer(
'encoder_%d' %
(int(len(self.model.layers) / 2) - 1)).output)
features = feature_model.predict(self.x)
km = KMeans(n_clusters=len(np.unique(self.y)),
n_init=20,
n_jobs=-1)
y_pred = km.fit_predict(features)
# print()
print(' ' * 8 + '|==> acc: %.4f, nmi: %.4f <==|' %
(metrics.acc(self.y, y_pred),
metrics.nmi(self.y, y_pred)))
cb.append(PrintACC(x, y))
# begin pretraining
t0 = time()
es = EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
patience=10)
cb.append(es)
self.autoencoder.fit(x,
x,
batch_size=batch_size,
epochs=epochs,
callbacks=cb)
print('Pretraining time: ', time() - t0)
self.autoencoder.save_weights(save_dir + '/ae_weights.h5')
print('Pretrained weights are saved to %s/ae_weights.h5' % save_dir)
self.pretrained = True
def train(model, data, args):
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
# callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=int(args.debug))
checkpoint = callbacks.ModelCheckpoint(args.save_dir +
'/weights-{epoch:02d}.hdf5',
monitor='val_capsnet_acc',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (args.lr_decay**epoch))
# compile the model
model.compile(
optimizer=optimizers.Adam(lr=args.lr),
loss=[margin_loss, reconstruction_loss
], # We scale down this reconstruction loss by 0.0005 so that
loss_weights=[
1., args.scale_reconstruction_loss
], # ...it does not dominate the margin loss during training.
metrics={'capsnet': 'accuracy'})
# Generator with data augmentation as used in [1]
def train_generator_with_augmentation(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(
width_shift_range=shift_fraction,
height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
yield ([x_batch, y_batch], [y_batch, x_batch])
generator = train_generator_with_augmentation(x_train, y_train,
args.batch_size,
args.shift_fraction)
model.fit_generator(
generator=generator,
steps_per_epoch=int(y_train.shape[0] / args.batch_size),
epochs=args.epochs,
validation_data=[
[x_test, y_test], [y_test, x_test]
], # Note: For the decoder the input is the label and the output the image
callbacks=[log, tb, checkpoint, lr_decay])
model.save_weights(args.save_dir + '/trained_model.hdf5')
print('Trained model saved to \'%s/trained_model.hdf5\'' % args.save_dir)
utils.plot_log(args.save_dir + '/log.csv', show=True)
return model
def main():
# Create the model.
model = SegNetSkip(input_shape=(320, 320), classes=FLAGS.numClasses)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(lr=0.01),
metrics=['acc'])
if FLAGS.saveModel:
# serialize model to JSON
model_json = model.to_json()
with open("weights/%s_model.json" % (FLAGS.experimentName),
"w") as json_file:
json_file.write(model_json)
# Read the dataset.
train_generator = data_gen(FLAGS.trainImageDir, FLAGS.trainLabelsDir,
FLAGS.numClasses)
validation_generator = data_gen(FLAGS.valImagesDir, FLAGS.valLabelsDir,
FLAGS.numClasses)
# Create the CSV Logger callback
csv_logger = callbacks.CSVLogger(
'logs/%s_training_%s.log' %
(FLAGS.experimentName, datetime.datetime.now()))
reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=4,
min_lr=0.00003,
verbose=1)
checkpoint = callbacks.ModelCheckpoint(
'weights/%s_weights.{epoch:02d}-{val_loss:.2f}.hdf5' %
(FLAGS.experimentName),
monitor='val_loss',
verbose=1,
period=5)
# Train the model.
print('Started training.')
start_time = time.time()
history = model.fit_generator(
train_generator,
steps_per_epoch=400,
epochs=60,
validation_data=validation_generator,
validation_steps=150,
callbacks=[csv_logger, reduce_lr, checkpoint])
print('Train took: %s' % (time.time() - start_time))
if FLAGS.saveFinalWeights:
# serialize weights to HDF5
model.save_weights("weights/%s_model.h5" % (FLAGS.experimentName))
print("Saved model to disk.")
def train(model, data, args):
# unpacking the data
(x_train, y_train), (x_val, y_val) = data
def train_generator(x, y, batch_size):
crop_length = 72
train_datagen = ImageDataGenerator()
# make a pseudo RGB image
x = np.concatenate((x[:, :, :, :], x[:, :, :, 0:1]), -1)
generator = train_datagen.flow(x, y, batch_size=batch_size)
while True:
batch_x, batch_y = generator.next()
batch_x = batch_x[:, :, :, 0:2]
batch_x = random_brightness(batch_x, (0.2, 1.8))
batch_x = random_contrast(batch_x, (0.3, 3))
batch_crops = np.zeros(
(batch_x.shape[0], crop_length, crop_length, batch_x.shape[3]))
for i in range(batch_x.shape[0]):
batch_crops[i] = random_crop(batch_x[i],
(crop_length, crop_length))
yield (batch_crops, batch_y)
log = callbacks.CSVLogger(args.save_dir + '/log.csv')
tb = TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=int(args.debug))
checkpoint = callbacks.ModelCheckpoint(args.save_dir +
'/weights-{epoch:02d}.h5',
monitor='val_acc',
save_best_only=True,
save_weights_only=True,
verbose=1)
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss=generalized_kullback_leibler_divergence,
metrics={'lvq_caps': 'accuracy'})
model.fit_generator(
generator=train_generator(x_train, y_train, args.batch_size),
steps_per_epoch=int(y_train.shape[0] / args.batch_size),
epochs=args.epochs,
validation_data=[center_crop(x_val), y_val],
callbacks=[log, checkpoint, tb])
model.save_weights(args.save_dir + '/trained_model.h5')
print('Trained model saved to \'%s/trained_model.h5\'' % args.save_dir)
return model
def main():
import os
import snorbdata
from keras.datasets import cifar10, cifar100
# setting the hyper parameters
args = {'epochs':50, 'batch_size':250, 'lr': 1e-3, 'decay': 0.8, 'iters': 3, 'weights': None, 'save_dir':'./results', 'dataset': 10}
print(args)
if not os.path.exists(args['save_dir']):
os.makedirs(args['save_dir'])
# load data
# define model
graph = tf.Graph()
with graph.as_default():
tf.add_check_numerics_ops()
if args['dataset'] == 10 or args['dataset'] == 100:
model = CapsNet_EM(input_shape=(32, 32, 3), num_classes=args['dataset'], iters=args['iters'], cifar=True, num_caps=(16, 24, 24))
else:
model = CapsNet_EM(input_shape=(args['dataset'], args['dataset'], 1), num_classes=5, iters=args['iters'])
print('-'*30 + 'Summary for Model' + '-'*30)
model.summary()
print('-'*30 + 'Summaries Done' + '-'*30)
if args['dataset'] == 10:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train, y_test = np.eye(10)[np.squeeze(y_train)], np.eye(10)[np.squeeze(y_test)]
elif args['dataset'] == 100:
(x_train, y_train), (x_test, y_test) = cifar100.load_data()
y_train, y_test = np.eye(100)[np.squeeze(y_train)], np.eye(100)[np.squeeze(y_test)]
else:
x_train, y_train, x_test, y_test = snorbdata.load_data()
if len(x_train.shape) < 4:
x_train = np.expand_dims(x_train, axis=-1)
if len(x_test.shape) < 4:
x_test = np.expand_dims(x_test, axis=-1)
print('Done loading data')
# init the model weights with provided one
if args['weights'] is not None:
model.load_weights(args['weights'])
log = callbacks.CSVLogger(args['save_dir'] + '/log.csv')
tb = callbacks.TensorBoard(log_dir=args['save_dir'] + '/tensorboard-logs', batch_size=args['batch_size'],
write_graph=True, write_images=True)
checkpoint = callbacks.ModelCheckpoint(args['save_dir'] + '/w_{epoch:02d}.h5', monitor='val_categorical_accuracy',
save_best_only=True, save_weights_only=True, verbose=1, period=2)
lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: args['lr'] * args['decay']**epoch)
naan = callbacks.TerminateOnNaN()
# compile and train model
for e in range(args['epochs']):
model.compile(optimizer=optimizers.Nadam(lr=args['lr']), loss=spread_loss_wrap(e, 0.2, 0.1, args['batch_size']), \
metrics=['categorical_accuracy'])
train_gen = ImageDataGenerator().flow(x_train, y_train, batch_size=args['batch_size'])
test_gen = ImageDataGenerator().flow(x_test, y_test, batch_size=args['batch_size'])
model.fit_generator(train_gen, validation_data=test_gen, initial_epoch=e, epochs=e +1, verbose=1, callbacks=[log, tb, checkpoint, lr_decay, naan])
model.save_weights(args['save_dir'] + '/model.h5')
print('Trained model saved to \'%s' % args['save_dir'])
return
def train(model, args):
# Define callbacks
log = callbacks.CSVLogger(args.save_dir + '/log.csv', append=True)
tb = callbacks.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs',
batch_size=args.batch_size,
histogram_freq=int(args.debug))
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: args.lr * (args.lr_decay**epoch))
checkpoint = callbacks.ModelCheckpoint(args.save_dir +
'/weights-{epoch:02d}.h5',
monitor='val_capsnet_acc',
save_best_only=True,
save_weights_only=True,
verbose=args.verbose)
early_stopper = callbacks.EarlyStopping(monitor='val_capsnet_loss',
patience=args.patience,
verbose=args.verbose)
# Compile the model
model.compile(optimizer=optimizers.Adam(lr=args.lr),
loss=[margin_loss, 'mse'],
loss_weights=[1., args.lam_recon],
metrics={
'capsnet': [
'accuracy', top_3_categorical_accuracy,
'top_k_categorical_accuracy'
]
})
# Start training using custom generator
model.fit_generator(
generator=custom_generator(get_iterator(args.filepath,
args.input_size,
args.shift_fraction,
args.hor_flip,
args.whitening,
args.rotation_range,
args.brightness_range,
args.shear_range,
args.zoom_range,
subset="train"),
testing=args.testing),
steps_per_epoch=int(210000 / args.batch_size),
epochs=args.epochs,
validation_data=custom_generator(get_iterator(args.filepath,
subset="val"),
testing=args.testing),
validation_steps=int(40000 / args.batch_size),
callbacks=[log, tb, checkpoint, lr_decay, early_stopper],
initial_epoch=args.initial_epoch)
# Save the model
model_path = '/t_model.h5'
model.save(args.save_dir + model_path)
print('The model saved to \'%s' + model_path + '\'' % args.save_dir)
def main():
now = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
model_name = 'pretrain_NASNet_' + now + '.h5'
batch_size = 32
num_epochs = 30
lr = .0001
num_train_samples = len(os.listdir('./data/train/cancer')) + len(os.listdir('./data/train/healthy'))
num_valid_samples = len(os.listdir('./data/validation/cancer')) + len(os.listdir('./data/validation/healthy'))
# Build our model
input_tensor = Input(shape=(96, 96, 3))
NASNet = NASNetMobile(include_top=False, input_shape=(96, 96, 3))
x = NASNet(input_tensor)
x1 = layers.GlobalMaxPooling2D()(x)
x2 = layers.GlobalAveragePooling2D()(x)
x3 = layers.Flatten()(x)
z = layers.Concatenate(axis=-1)([x1, x2, x3])
z = layers.Dropout(.5)(z)
output_tensor = layers.Dense(1, activation='sigmoid')(z)
model = Model(input_tensor, output_tensor)
model.summary()
# Get things ready to train: tweak learning rate, etc.
model.compile(optimizer=Adam(lr), loss='binary_crossentropy', metrics=['acc'])
train_generator = train_gen(batch_size)
validation_generator = valid_gen(batch_size)
steps_per_epoch = num_train_samples / batch_size
validation_steps = num_valid_samples / batch_size
# Basic callbacks
checkpoint = callbacks.ModelCheckpoint(filepath='./models/' + model_name,
monitor='val_loss',
save_best_only=True)
early_stop = callbacks.EarlyStopping(monitor='val_acc',
patience=4)
csv_logger = callbacks.CSVLogger('./logs/' + model_name.split('.')[0] + '.csv')
callback_list = [checkpoint, early_stop, csv_logger]
# Training begins
history = model.fit_generator(train_generator,
steps_per_epoch=steps_per_epoch,
epochs=num_epochs,
verbose=1,
callbacks=callback_list,
validation_data=validation_generator,
validation_steps=validation_steps)
model.save('./models/' + model_name)
make_plots(history, model_name)
def get_callbacks(save_folder, save_frequency):
# Define save path
save_path = os.path.join(save_folder, '{epoch}.h5')
log_path = os.path.join(save_folder, 'training_log.csv')
# Callbacks
checkpoint = callbacks.ModelCheckpoint(save_path, period=save_frequency)
logger = callbacks.CSVLogger(log_path)
lr_reduce = callbacks.ReduceLROnPlateau(factor=0.5, patience=5, min_lr=1e-5)
return [checkpoint, logger, lr_reduce]
def train():
print('-'*30)
print('Loading and preprocessing train data...')
print('-'*30)
imgs_train, imgs_gtruth_train = load_train_data()
print('-'*30)
print('Loading and preprocessing validation data...')
print('-'*30)
imgs_val, imgs_gtruth_val = load_validatation_data()
print('-'*30)
print('Creating and compiling model...')
print('-'*30)
# create a model
model = unet_model_3d(input_shape=config["input_shape"],
depth=config["depth"],
pool_size=config["pool_size"],
n_labels=config["n_labels"],
initial_learning_rate=config["initial_learning_rate"],
deconvolution=config["deconvolution"])
model.summary()
print('-'*30)
print('Fitting model...')
print('-'*30)
#============================================================================
print('training starting..')
log_filename = 'outputs/' + image_type +'_model_train.csv'
csv_log = callbacks.CSVLogger(log_filename, separator=',', append=True)
# early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='min')
#checkpoint_filepath = 'outputs/' + image_type +"_best_weight_model_{epoch:03d}_{val_loss:.4f}.hdf5"
checkpoint_filepath = 'outputs/' + 'weights.h5'
checkpoint = callbacks.ModelCheckpoint(checkpoint_filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
callbacks_list = [csv_log, checkpoint]
callbacks_list.append(ReduceLROnPlateau(factor=config["learning_rate_drop"], patience=config["patience"],
verbose=True))
callbacks_list.append(EarlyStopping(verbose=True, patience=config["early_stop"]))
#============================================================================
hist = model.fit(imgs_train, imgs_gtruth_train, batch_size=config["batch_size"], nb_epoch=config["n_epochs"], verbose=1, validation_data=(imgs_val,imgs_gtruth_val), shuffle=True, callbacks=callbacks_list) # validation_split=0.2,
model_name = 'outputs/' + image_type + '_model_last'
model.save(model_name) # creates a HDF5 file 'my_model.h5'
def train(model, data, epoch_size_frac=1.0, epochs=1):
"""
Training a CapsuleNet
:param model: the CapsuleNet model
:param data: a tuple containing training and testing data, like `((x_train, y_train), (x_test, y_test))`
:param args: arguments
:return: The trained model
"""
# unpacking the data
(x_train, y_train), (x_test, y_test) = data
# callbacks
log = callbacks.CSVLogger('log.csv')
checkpoint = callbacks.ModelCheckpoint('weights-{epoch:02d}.h5',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: 0.001 * np.exp(-epoch / 10.))
# compile the model
model.compile(optimizer='adam',
loss=[margin_loss, 'mse'],
loss_weights=[1., 0.0005],
metrics={'out_caps': 'accuracy'})
"""
# Training without data augmentation:
model.fit([x_train, y_train], [y_train, x_train], batch_size=args.batch_size, epochs=args.epochs,
validation_data=[[x_test, y_test], [y_test, x_test]], callbacks=[log, tb, checkpoint])
"""
# -----------------------------------Begin: Training with data augmentation -----------------------------------#
def train_generator(x, y, batch_size, shift_fraction=0.):
train_datagen = ImageDataGenerator(
width_shift_range=shift_fraction,
height_shift_range=shift_fraction) # shift up to 2 pixel for MNIST
generator = train_datagen.flow(x, y, batch_size=batch_size)
while 1:
x_batch, y_batch = generator.next()
yield ([x_batch, y_batch], [y_batch, x_batch])
# Training with data augmentation. If shift_fraction=0., also no augmentation.
model.fit_generator(generator=train_generator(x_train, y_train, 64, 0.1),
steps_per_epoch=int(epoch_size_frac *
y_train.shape[0] / 64),
epochs=epochs,
validation_data=[[x_test, y_test], [y_test, x_test]],
callbacks=[log, checkpoint, lr_decay])
# -----------------------------------End: Training with data augmentation -----------------------------------#
model.save_weights('trained_model.h5')
print('Trained model saved to \'trained_model.h5\'')
return model
def train(dataset='cifar10', name="bn",**kwargs):
batch_size = kwargs['batch_size'] if 'batch_size' in kwargs else 128
epochs = kwargs['epochs'] if 'epochs' in kwargs else 300
dropout = kwargs['dropout'] if 'dropout' in kwargs else 0.5
weight_decay = kwargs['weight_decay'] if 'weight_decay' in kwargs else 0.0001
log_filepath = './'+name
models = {"bn":build_model_bn, "nonBn":build_model_nonBn}
schedulers = {"bn":scheduler_bn, "nonBn":scheduler_nonBn}
# load data
if not 'data' in kwargs:
(x_train, y_train), (x_test, y_test), (img_rows, img_cols, num_classes) = datama.getData(dataset)
else:
((x_train, y_train), (x_test, y_test), num_classes) = kwargs['data']
img_rows, img_cols = x_train.shape[1], x_train.shape[2]
# build network
model = models[name](x_train.shape[1:],dropout,weight_decay)
print(model.summary())
change_lr = LearningRateScheduler(schedulers[name])
if not 'logfile' in kwargs:
csvlog = callbacks.CSVLogger("./log/netinnet_" + dataset + ".log", separator=',', append=False)
else:
csvlog = callbacks.CSVLogger(kwargs['logfile'], separator=',', append=False)
if not 'bestModelfile' in kwargs:
checkPoint = callbacks.ModelCheckpoint('./model/netinnet_' + dataset + ".h5", save_best_only=True, monitor="val_acc", verbose=1)
else:
checkPoint = callbacks.ModelCheckpoint(kwargs['bestModelfile'], monitor="val_acc",
save_best_only=True, verbose=1)
cbks = [change_lr,csvlog, checkPoint]
# set data augmentation
# if you do not want to use data augmentation, comment below codes.
print('Using real-time data augmentation.')
datagen = ImageDataGenerator(horizontal_flip=True, width_shift_range=0.125, height_shift_range=0.125,
fill_mode='constant', cval=0.)
datagen.fit(x_train)
iterations = x_train.shape[0]//batch_size
#start training
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=iterations,
epochs=epochs, callbacks=cbks, validation_data=(x_test, y_test))
def model_fitting(self):
csv_path = ROOT_PATH + 'result_output/' + self.model_h5_name + '_train_log.log'
csv_logger = callbacks.CSVLogger(csv_path, append=True)
self.model.fit(self.X_train, self.Y_train,
batch_size=self.batch_size,
epochs=self.nb_epoch,
verbose=2,
# validation_data=(self.X_test, self.Y_test),
callbacks=[csv_logger])
self.model.save(ROOT_PATH + self.model_h5_name + '_model_weights.h5')
def train(epochs=200, batch_size=64, mode=1):
import numpy as np
import os
from keras import callbacks
from keras.utils.vis_utils import plot_model
if mode == 1:
num_classes = 10
(x_train, y_train), (x_test, y_test) = load_cifar_10()
else:
num_classes = 100
(x_train, y_train), (x_test, y_test) = load_cifar_100()
model = CapsNetv1(input_shape=[32, 32, 3], n_class=num_classes, n_route=3)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
model.summary()
log = callbacks.CSVLogger('results/capsule-cifar-' + str(num_classes) +
'-log.csv')
tb = callbacks.TensorBoard(log_dir='results/tensorboard-capsule-cifar-' +
str(num_classes) + '-logs',
batch_size=batch_size,
histogram_freq=True)
checkpoint = callbacks.ModelCheckpoint(
'weights/capsule-cifar-' + str(num_classes) + 'weights-{epoch:02d}.h5',
save_best_only=True,
save_weights_only=True,
verbose=1)
lr_decay = callbacks.LearningRateScheduler(
schedule=lambda epoch: 0.001 * np.exp(-epoch / 10.))
plot_model(model,
to_file='models/capsule-cifar-' + str(num_classes) + '.png',
show_shapes=True)
model.compile(optimizer=optimizers.Adam(lr=0.001),
loss=[margin_loss, 'mse'],
loss_weights=[1., 0.1],
metrics={
'output_recon': 'accuracy',
'output': 'accuracy'
})
from utils.helper_function import data_generator
generator = data_generator(x_train, y_train, batch_size)
# Image generator significantly increase the accuracy and reduce validation loss
model.fit_generator(generator,
steps_per_epoch=x_train.shape[0] // batch_size,
validation_data=([x_test, y_test], [y_test, x_test]),
epochs=epochs,
verbose=1,
max_q_size=100,
callbacks=[log, tb, checkpoint, lr_decay])
def get_callbacks(self, opt):
# ModelCheckpoints: saving model after each epoch
fn1 = (os.path.basename(self.model_params_file_path).replace(
'.json', ''))
fn = (f'{fn1}___{self.start_time}'
f'___model_%s{"_TEST" if opt.test else ""}.h5' % ('{epoch:02d}'))
filepath = os.path.join(self.path_nn_model, self.model.name, fn)
del fn
checkpoint = callbacks.ModelCheckpoint(filepath,
monitor='val_loss',
verbose=opt.verbose)
# TerminateOnNaN
tonan = callbacks.TerminateOnNaN()
# History
history = callbacks.History()
# CSV logger: saves epoch train and valid loss to a log file
fn1 = (os.path.basename(self.model_params_file_path).replace(
'.json', ''))
fn = (f'{fn1}___{self.start_time}'
f'___training{"_TEST" if opt.test else ""}.log')
filepath = os.path.join(self.path_nn_model, self.model.name, fn)
csv_logger = callbacks.CSVLogger(filepath, separator=',', append=True)
# Learning rate scheduler
def exp_decay(epoch,
initial_lrate=self.model_params['keras_train']['lr'],
decay=self.model_params['keras_train']['lr_decay']):
lrate = initial_lrate * np.exp(-decay * epoch)
return lrate
def learning_rate_decay(
epoch,
initial_lrate=self.model_params['keras_train']['lr'],
decay=self.model_params['keras_train']['lr_decay']):
lrate = initial_lrate * (1 - decay)**epoch
return lrate
lrs = callbacks.LearningRateScheduler(learning_rate_decay)
callbacks_list = [
tonan, checkpoint, history, csv_logger, csv_logger, lrs
]
# Early stopping: stops training if validation loss does not improves
if (self.model_params['keras_train'].get('early_stopping_n')
is not None):
es = callbacks.EarlyStopping(
monitor='val_loss',
min_delta=0,
patience=self.model_params['keras_train']['early_stopping_n'],
verbose=opt.verbose)
callbacks_list.append(es)
return callbacks_list
def build_callbacks(conf):
'''
The purpose of the method is to set up logging and history. It is based on
Keras Callbacks
https://github.com/fchollet/keras/blob/fbc9a18f0abc5784607cd4a2a3886558efa3f794/keras/callbacks.py
Currently used callbacks include: BaseLogger, CSVLogger, EarlyStopping.
Other possible callbacks to add in future:
RemoteMonitor, LearningRateScheduler
Argument list:
- conf: There is a "callbacks" section in conf.yaml file.
Relevant parameters are:
list: Parameter specifying additional callbacks, read in the driver
script and passed as an argument of type list (see next arg)
metrics: List of quantities monitored during training and
validation
mode: one of {auto, min, max}. The decision to overwrite the
current save file is made based on either the maximization or the
minimization of the monitored quantity. For val_acc, this should be max,
for val_loss this should be min, etc. In auto mode, the direction is
automatically inferred from the name of the monitored quantity.
monitor: Quantity used for early stopping, has to be from the list
of metrics
patience: Number of epochs used to decide on whether to apply early
stopping or continue training
- callbacks_list: uses callbacks.list configuration parameter,
specifies the list of additional callbacks
Returns:
modified list of callbacks
'''
# mode = conf['callbacks']['mode']
# monitor = conf['callbacks']['monitor']
# patience = conf['callbacks']['patience']
csvlog_save_path = conf['paths']['csvlog_save_path']
# CSV callback is on by default
if not os.path.exists(csvlog_save_path):
os.makedirs(csvlog_save_path)
# callbacks_list = conf['callbacks']['list']
callbacks = [cbks.BaseLogger()]
callbacks += [
cbks.CSVLogger("{}callbacks-{}.log".format(
csvlog_save_path,
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))
]
return cbks.CallbackList(callbacks)
def test_stop_training_csv(tmpdir):
np.random.seed(1337)
fp = str(tmpdir / 'test.csv')
(X_train, y_train), (X_test,
y_test) = get_test_data(num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim, ),
classification=True,
num_classes=num_classes)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
cbks = [callbacks.TerminateOnNaN(), callbacks.CSVLogger(fp)]
model = Sequential()
for _ in range(5):
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_classes, activation='linear'))
model.compile(loss='mean_squared_error', optimizer='rmsprop')
def data_generator():
i = 0
max_batch_index = len(X_train) // batch_size
tot = 0
while 1:
if tot > 3 * len(X_train):
yield np.ones([batch_size, input_dim]) * np.nan, np.ones(
[batch_size, num_classes]) * np.nan
else:
yield (X_train[i * batch_size:(i + 1) * batch_size],
y_train[i * batch_size:(i + 1) * batch_size])
i += 1
tot += 1
i = i % max_batch_index
history = model.fit_generator(data_generator(),
len(X_train) // batch_size,
validation_data=(X_test, y_test),
callbacks=cbks,
epochs=20)
loss = history.history['loss']
assert len(loss) > 1
assert loss[-1] == np.inf or np.isnan(loss[-1])
values = []
with open(fp) as f:
for x in reader(f):
values.append(x)
assert 'nan' in values[-1], 'The last epoch was not logged.'
os.remove(fp)
