def train_and_predict(model_type, gpus):
"""
Trains model and makes predictions file
"""
# creating data generators
train_datagen = ImageDataGenerator(rescale=1. / 255, horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
TRAIN_DATA_PATH,
class_mode='binary',
seed=171717,
**config[model_type]['flow_generator'])
validation_generator = test_datagen.flow_from_directory(
VALIDATION_DATA_PATH,
class_mode='binary',
**config[model_type]['flow_generator'])
test_generator = test_datagen.flow_from_directory(
TEST_DATA_PATH,
class_mode=None,
classes=CLASSES,
shuffle=False,
**config[model_type]['flow_generator'])
# loading the model
parallel_model, model = get_model(model=model_type, gpus=gpus)
print('Training model')
print(model.summary())
history = parallel_model.fit_generator(
train_generator,
validation_data=validation_generator,
callbacks=[
EarlyStopping(monitor='val_loss', min_delta=0, patience=5),
ReduceLROnPlateau(
monitor='val_loss', factor=0.2, patience=3, min_lr=0.000001),
TerminateOnNaN()
],
max_queue_size=100,
use_multiprocessing=True,
workers=cpu_count(),
**config[model_type]['fit_generator'])
# history of training
# print(history.history.keys())
# Saving architecture + weights + optimizer state
model_path = path.join(MODELS_PATH, '{}_{:.4f}_{:.4f}'.format(
model_type, history.history['val_loss'][-1]
if 'val_loss' in history.history else history.history['loss'][-1],
history.history['val_acc'][-1]
if 'val_acc' in history.history else history.history['acc'][-1]))
try_makedirs(model_path)
plot_model(model, path.join(model_path, 'model.png'), show_shapes=True)
plot_loss_acc(history, model_path)
print('Saving model')
model.save(path.join(model_path, 'model.h5'))
# Building confusion matrices for every class for validation data
print("Building confusion matrices")
val_preds = model.predict_generator(
validation_generator,
max_queue_size=100,
use_multiprocessing=True,
workers=cpu_count())
plot_confusion_matrix(
confusion_matrix(
list(validation_generator.classes), np.argmax(val_preds, axis=1)),
CLASSES, model_path)
print('Generating predictions')
predictions = model.predict_generator(
test_generator,
max_queue_size=100,
use_multiprocessing=True,
workers=cpu_count())
pred_classes = np.argmax(predictions)
# Dealing with missing data
ids = list(map(lambda id: id[5:-4], test_generator.filenames))
proba = predictions[np.arange(len(predictions)), pred_classes]
# Generating predictions.csv for Kaggle
pd.DataFrame({
'id': ids,
'predicted': pred_classes,
}).sort_values(by='id').to_csv(
path.join(model_path, 'predictions.csv'), index=False)
# Generating predictions.csv with some additional data for post-processing
pd.DataFrame({
'id': ids,
'predicted': pred_classes,
'proba': proba
}).sort_values(by='id').to_csv(
path.join(model_path, 'predictions_extd.csv'), index=False)
model.float()
# optimizer
if optim == 'sgd':
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
momentum=sgd_momentum)
else:
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
betas=adam_beta,
eps=adam_epsilon)
scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.01)
weights = [10, 10, 100, 100, 100]
#weights = torch.nn.functional.normalize(torch.Tensor(weights), dim = 0)
# train
model, train_loss_epoch, train_acc_epoch, val_loss_epoch, val_acc_epoch = train_model(
model, average_difference_loss, location_success_count, color_success_count, \
optimizer, scheduler, train_dataloader, test_dataloader, weights, N, num_of_param, epoch)
plot_loss_acc(train_loss_epoch, train_acc_epoch, val_loss_epoch,
val_acc_epoch)
if epoch > 1:
torch.save(
model,
os.path.join(
'checkpoint',
datetime.now().strftime("_%d-%m-%Y_%H_%M_%S") + 'model'))
valid_loss_list = []
valid_acc_list = []
best_acc = 0
best_epoch = -1
for epoch in range(1, args.epochs + 1):
train_loss, train_acc = train(epoch, model, train_iterator, optimizer, criterion)
train_loss_list.append(train_loss)
train_acc_list.append(train_acc)
valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
valid_loss_list.append(valid_loss)
valid_acc_list.append(valid_acc)
msg = '...Epoch %02d, val loss = %.4f, val acc = %.4f' % (
epoch, valid_loss, valid_acc
)
LOG_INFO(msg)
if valid_acc > best_acc:
best_acc = valid_acc
best_epoch = epoch
torch.save(model.state_dict(), 'best-model-%s.pth' % args.cell_type)
LOG_INFO('Test best model @ Epoch %02d' % best_epoch)
model.load_state_dict(torch.load('best-model-%s.pth' % args.cell_type))
test_loss, test_acc = evaluate(model, test_iterator, criterion)
LOG_INFO('Finally, test loss = %.4f, test acc = %.4f' % (test_loss, test_acc))
plot_loss_acc(args.epochs, train_acc_list, valid_acc_list, train_loss_list, valid_loss_list, "GRUCell_None")
print ('sets dimensions: train x, train y, val x, val y, test x, test y')
print (x_train.shape, y_train.shape, x_val.shape, y_val.shape, x_test.shape, y_test.shape)
# Hyperparameters
batch_size = settings.batch_size
dropout_rate = settings.dropout_rate
learning_rate = settings.learning_rate
n_feats = x_train.shape[1]
epochs = settings.epochs
nn_type = settings.nn_type # feed_forward only one supported right now
# build model
model = build_feedforward(n_feats, dropout_rate, learning_rate)
# train model
history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=epochs, verbose=2, batch_size=batch_size)
# evaluate model
_, train_acc = model.evaluate(x_train, y_train, verbose=0)
_, test_acc = model.evaluate(x_test, y_test, verbose=0)
print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
# plot loss during training
plot_loss_acc(history, settings.model_name)
# serialize model to JSON
model_json = model.to_json()
with open(settings.model_name+".json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights(settings.model_name+".h5")
print("saved model")
def main():
print_title('[PDR] Train Model', symbol='*')
print_title('@author: Kefeng Zhu ([email protected], [email protected])', '')
print_title('Date: %s' % config.date, '')
# 1. Make dirs/Show parameters
if not os.path.exists(config.model_save_dir):
os.mkdir(config.model_save_dir)
# Print and save parameteres
print_title('I. Training Parameters')
config.print_params()
# 2. Build and compile model
print_title("II. Build Model")
print('[KF INFO] Start building model ...')
model = build_model(config.model_name, config.img_size, config.num_classes)
optimizer = optimizers.Adam(lr=config.lr)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
print('[KF INFO] Building model complete!')
# 3. Callbacks
callbacks = []
callbacks.append(
CSVLogger(os.path.join(config.model_save_dir, 'train_history.csv'),
append=True,
separator=';'))
callbacks.append(
ModelCheckpoint(os.path.join(config.model_save_dir,
config.model_filename),
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto'))
if config.early_stopping_patience:
callbacks.append(
EarlyStopping(monitor='val_loss',
patience=config.early_stopping_patience,
verbose=1,
mode='auto'))
if config.lr_reduce_patience:
callbacks.append(
ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
verbose=1,
patience=config.lr_reduce_patience))
# 4. Data generators
print_title('III. Data Generator')
print('[KF INFO] Initializing data generator ...')
train_gen = ImageDataGenerator(rescale=1. / 255,
rotation_range=24,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
fill_mode='nearest')
train_flow = train_gen.flow_from_directory(config.train_data_dir,
target_size=(config.img_size,
config.img_size),
batch_size=config.batch_size,
class_mode='categorical')
print('[KF INFO] training data flow from %s' % config.train_data_dir)
val_gen = ImageDataGenerator(rescale=1. / 255)
val_flow = val_gen.flow_from_directory(config.val_data_dir,
target_size=(config.img_size,
config.img_size),
batch_size=config.batch_size,
class_mode='categorical')
print('[KF INFO] validation data flow from %s' % config.val_data_dir)
# Save class indices
with open(
os.path.join(config.model_save_dir, config.class_indices_filename),
'w') as f:
json.dump(train_flow.class_indices, f, indent=4)
print('class_indices is saved to file: %s' %
config.class_indices_filename)
print('[KF INFO] Initializing data generator complete!')
# 5. Train
print_title('IV. Train Model')
print('[KF INFO] Start Training ...')
start = time.time()
H = model.fit_generator(
train_flow,
validation_data=val_flow,
epochs=config.epochs,
steps_per_epoch=config.train_size // config.batch_size,
validation_steps=config.val_size // config.batch_size,
callbacks=callbacks)
# Save training time:
end = time.time()
print('[KF INFO] Total training time :', end - start)
print('[KF INFO] Training complete!')
# 6. Save metric and plot
print_title('V. Save Results')
print('[KF INFO] Start saving results ...')
# [Pitfall] Force casting float to all elements in the dictionary,
# to prevent the "np.float32 cannot be Jsonity" problem.
for k in H.history.keys():
for i in range(len(H.history[k])):
H.history[k][i] = float(H.history[k][i])
# save
with open(os.path.join(config.model_save_dir, config.history_filename),
'w') as f:
json.dump(H.history, f, indent=4)
print('[KF INFO] History is saved as "%s"' % config.history_filename)
# load
with open(os.path.join(config.model_save_dir, config.history_filename),
'r') as f:
h = json.load(f)
plot_loss_acc(h,
config.model_save_dir,
filename=config.plt_acc_loss_filename,
title=config.prefix)
if h.get('lr'):
plot_lr(h,
config.model_save_dir,
filename=config.plt_lr_filename,
title=config.prefix)
print('[KF INFO] Saving results complete!')
def plot(self):
utils.plot_loss_acc(self.total_loss, self.total_accy)
start_time = time.time()
for epoch in range(1, EPOCHS + 1):
train_loss, train_acc = train(model, device, train_loader, optimizer,
epoch)
test_loss, test_acc = test(model, device, test_loader)
# save train & test loss & accuracy
train_loss_list.append(train_loss)
train_acc_list.append(train_acc)
test_loss_list.append(test_loss)
test_acc_list.append(test_acc)
end_time = time.time()
plot_loss_acc(EPOCHS,
train_loss_list,
test_loss_list,
'CNN',
'with BN',
loss=True)
plot_loss_acc(EPOCHS,
train_acc_list,
test_acc_list,
'CNN',
'with BN',
loss=False)
print('Time per epoch = {:.2f}s'.format((end_time - start_time) / EPOCHS))
# with BN: 14.90s per epoch
# without BN: 12.92s per epoch
train_acc_list.append(train_acc)
LOG_INFO('Testing @ %d epoch...' % (epoch))
test_loss, test_acc, visual = test_net(model, loss, test_data, test_label,
config['batch_size'], epoch,
config['layer_vis'])
# save loss & accuracy data for test epoch
test_loss_list.append(test_loss)
test_acc_list.append(test_acc)
# plot visual during testing of the last epoch
if epoch == (config['max_epoch'] - 1):
plot_visual = visual[0:25, :, :]
end_time = time.time()
plot_loss_acc(config['max_epoch'],
train_loss_list,
test_loss_list,
model.name,
loss.name,
loss=True)
plot_loss_acc(config['max_epoch'],
train_acc_list,
test_acc_list,
model.name,
loss.name,
loss=False)
plot_vis(plot_visual, model.name, loss.name, config['layer_vis'])
print('Time per epoch = ' +
str((end_time - start_time) / config['max_epoch']) + 's')
def run_training(nb_epoch, model_filename, plot_path):
"""
Training the Fashion MNIST model
:param nb_epoch:
:return:
"""
# ===================================================================
# Create the model
# ===================================================================
model = Sequential()
model.add(
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=[28, 28, 1]))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3)))
model.add(Flatten())
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
# ===================================================================
# Compile the model
# ===================================================================
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
# ===================================================================
# Train the model
# ===================================================================
print("[i] training the model...")
history = model.fit_generator(data_generator(),
steps_per_epoch=60000 // 32,
nb_epoch=nb_epoch,
validation_data=None)
# ===================================================================
# Save the best model
# ===================================================================
if not os.path.exists("models/"):
print("[i] creating the [models/] directory...")
os.makedirs("models/")
else:
print("[i] the [models/] directory already exists!")
print("[i] saving the model...")
model.save_weights('./models/' + model_filename)
json_string = model.to_json()
with open('./models/config.json', 'w') as f:
f.write(json_string)
# ===================================================================
# Plot the training loss and accuracy
# ===================================================================
plot_loss_acc(history, nb_epoch, plot_path)
print("********************* Done **********************")