def train_and_predict(model_type, gpus): """ Trains model and makes predictions file """ # creating data generators train_datagen = ImageDataGenerator(rescale=1. / 255, horizontal_flip=True) test_datagen = ImageDataGenerator(rescale=1. / 255) train_generator = train_datagen.flow_from_directory( TRAIN_DATA_PATH, class_mode='binary', seed=171717, **config[model_type]['flow_generator']) validation_generator = test_datagen.flow_from_directory( VALIDATION_DATA_PATH, class_mode='binary', **config[model_type]['flow_generator']) test_generator = test_datagen.flow_from_directory( TEST_DATA_PATH, class_mode=None, classes=CLASSES, shuffle=False, **config[model_type]['flow_generator']) # loading the model parallel_model, model = get_model(model=model_type, gpus=gpus) print('Training model') print(model.summary()) history = parallel_model.fit_generator( train_generator, validation_data=validation_generator, callbacks=[ EarlyStopping(monitor='val_loss', min_delta=0, patience=5), ReduceLROnPlateau( monitor='val_loss', factor=0.2, patience=3, min_lr=0.000001), TerminateOnNaN() ], max_queue_size=100, use_multiprocessing=True, workers=cpu_count(), **config[model_type]['fit_generator']) # history of training # print(history.history.keys()) # Saving architecture + weights + optimizer state model_path = path.join(MODELS_PATH, '{}_{:.4f}_{:.4f}'.format( model_type, history.history['val_loss'][-1] if 'val_loss' in history.history else history.history['loss'][-1], history.history['val_acc'][-1] if 'val_acc' in history.history else history.history['acc'][-1])) try_makedirs(model_path) plot_model(model, path.join(model_path, 'model.png'), show_shapes=True) plot_loss_acc(history, model_path) print('Saving model') model.save(path.join(model_path, 'model.h5')) # Building confusion matrices for every class for validation data print("Building confusion matrices") val_preds = model.predict_generator( validation_generator, max_queue_size=100, use_multiprocessing=True, workers=cpu_count()) plot_confusion_matrix( confusion_matrix( list(validation_generator.classes), np.argmax(val_preds, axis=1)), CLASSES, model_path) print('Generating predictions') predictions = model.predict_generator( test_generator, max_queue_size=100, use_multiprocessing=True, workers=cpu_count()) pred_classes = np.argmax(predictions) # Dealing with missing data ids = list(map(lambda id: id[5:-4], test_generator.filenames)) proba = predictions[np.arange(len(predictions)), pred_classes] # Generating predictions.csv for Kaggle pd.DataFrame({ 'id': ids, 'predicted': pred_classes, }).sort_values(by='id').to_csv( path.join(model_path, 'predictions.csv'), index=False) # Generating predictions.csv with some additional data for post-processing pd.DataFrame({ 'id': ids, 'predicted': pred_classes, 'proba': proba }).sort_values(by='id').to_csv( path.join(model_path, 'predictions_extd.csv'), index=False)
model.float() # optimizer if optim == 'sgd': optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=sgd_momentum) else: optimizer = optim.Adam(model.parameters(), lr=learning_rate, betas=adam_beta, eps=adam_epsilon) scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.01) weights = [10, 10, 100, 100, 100] #weights = torch.nn.functional.normalize(torch.Tensor(weights), dim = 0) # train model, train_loss_epoch, train_acc_epoch, val_loss_epoch, val_acc_epoch = train_model( model, average_difference_loss, location_success_count, color_success_count, \ optimizer, scheduler, train_dataloader, test_dataloader, weights, N, num_of_param, epoch) plot_loss_acc(train_loss_epoch, train_acc_epoch, val_loss_epoch, val_acc_epoch) if epoch > 1: torch.save( model, os.path.join( 'checkpoint', datetime.now().strftime("_%d-%m-%Y_%H_%M_%S") + 'model'))
valid_loss_list = [] valid_acc_list = [] best_acc = 0 best_epoch = -1 for epoch in range(1, args.epochs + 1): train_loss, train_acc = train(epoch, model, train_iterator, optimizer, criterion) train_loss_list.append(train_loss) train_acc_list.append(train_acc) valid_loss, valid_acc = evaluate(model, valid_iterator, criterion) valid_loss_list.append(valid_loss) valid_acc_list.append(valid_acc) msg = '...Epoch %02d, val loss = %.4f, val acc = %.4f' % ( epoch, valid_loss, valid_acc ) LOG_INFO(msg) if valid_acc > best_acc: best_acc = valid_acc best_epoch = epoch torch.save(model.state_dict(), 'best-model-%s.pth' % args.cell_type) LOG_INFO('Test best model @ Epoch %02d' % best_epoch) model.load_state_dict(torch.load('best-model-%s.pth' % args.cell_type)) test_loss, test_acc = evaluate(model, test_iterator, criterion) LOG_INFO('Finally, test loss = %.4f, test acc = %.4f' % (test_loss, test_acc)) plot_loss_acc(args.epochs, train_acc_list, valid_acc_list, train_loss_list, valid_loss_list, "GRUCell_None")
print ('sets dimensions: train x, train y, val x, val y, test x, test y') print (x_train.shape, y_train.shape, x_val.shape, y_val.shape, x_test.shape, y_test.shape) # Hyperparameters batch_size = settings.batch_size dropout_rate = settings.dropout_rate learning_rate = settings.learning_rate n_feats = x_train.shape[1] epochs = settings.epochs nn_type = settings.nn_type # feed_forward only one supported right now # build model model = build_feedforward(n_feats, dropout_rate, learning_rate) # train model history = model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=epochs, verbose=2, batch_size=batch_size) # evaluate model _, train_acc = model.evaluate(x_train, y_train, verbose=0) _, test_acc = model.evaluate(x_test, y_test, verbose=0) print('Train: %.3f, Test: %.3f' % (train_acc, test_acc)) # plot loss during training plot_loss_acc(history, settings.model_name) # serialize model to JSON model_json = model.to_json() with open(settings.model_name+".json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model.save_weights(settings.model_name+".h5") print("saved model")
def main(): print_title('[PDR] Train Model', symbol='*') print_title('@author: Kefeng Zhu ([email protected], [email protected])', '') print_title('Date: %s' % config.date, '') # 1. Make dirs/Show parameters if not os.path.exists(config.model_save_dir): os.mkdir(config.model_save_dir) # Print and save parameteres print_title('I. Training Parameters') config.print_params() # 2. Build and compile model print_title("II. Build Model") print('[KF INFO] Start building model ...') model = build_model(config.model_name, config.img_size, config.num_classes) optimizer = optimizers.Adam(lr=config.lr) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) model.summary() print('[KF INFO] Building model complete!') # 3. Callbacks callbacks = [] callbacks.append( CSVLogger(os.path.join(config.model_save_dir, 'train_history.csv'), append=True, separator=';')) callbacks.append( ModelCheckpoint(os.path.join(config.model_save_dir, config.model_filename), monitor='val_loss', verbose=1, save_best_only=True, mode='auto')) if config.early_stopping_patience: callbacks.append( EarlyStopping(monitor='val_loss', patience=config.early_stopping_patience, verbose=1, mode='auto')) if config.lr_reduce_patience: callbacks.append( ReduceLROnPlateau(monitor='val_loss', factor=0.5, verbose=1, patience=config.lr_reduce_patience)) # 4. Data generators print_title('III. Data Generator') print('[KF INFO] Initializing data generator ...') train_gen = ImageDataGenerator(rescale=1. / 255, rotation_range=24, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, vertical_flip=True, fill_mode='nearest') train_flow = train_gen.flow_from_directory(config.train_data_dir, target_size=(config.img_size, config.img_size), batch_size=config.batch_size, class_mode='categorical') print('[KF INFO] training data flow from %s' % config.train_data_dir) val_gen = ImageDataGenerator(rescale=1. / 255) val_flow = val_gen.flow_from_directory(config.val_data_dir, target_size=(config.img_size, config.img_size), batch_size=config.batch_size, class_mode='categorical') print('[KF INFO] validation data flow from %s' % config.val_data_dir) # Save class indices with open( os.path.join(config.model_save_dir, config.class_indices_filename), 'w') as f: json.dump(train_flow.class_indices, f, indent=4) print('class_indices is saved to file: %s' % config.class_indices_filename) print('[KF INFO] Initializing data generator complete!') # 5. Train print_title('IV. Train Model') print('[KF INFO] Start Training ...') start = time.time() H = model.fit_generator( train_flow, validation_data=val_flow, epochs=config.epochs, steps_per_epoch=config.train_size // config.batch_size, validation_steps=config.val_size // config.batch_size, callbacks=callbacks) # Save training time: end = time.time() print('[KF INFO] Total training time :', end - start) print('[KF INFO] Training complete!') # 6. Save metric and plot print_title('V. Save Results') print('[KF INFO] Start saving results ...') # [Pitfall] Force casting float to all elements in the dictionary, # to prevent the "np.float32 cannot be Jsonity" problem. for k in H.history.keys(): for i in range(len(H.history[k])): H.history[k][i] = float(H.history[k][i]) # save with open(os.path.join(config.model_save_dir, config.history_filename), 'w') as f: json.dump(H.history, f, indent=4) print('[KF INFO] History is saved as "%s"' % config.history_filename) # load with open(os.path.join(config.model_save_dir, config.history_filename), 'r') as f: h = json.load(f) plot_loss_acc(h, config.model_save_dir, filename=config.plt_acc_loss_filename, title=config.prefix) if h.get('lr'): plot_lr(h, config.model_save_dir, filename=config.plt_lr_filename, title=config.prefix) print('[KF INFO] Saving results complete!')
def plot(self): utils.plot_loss_acc(self.total_loss, self.total_accy)
start_time = time.time() for epoch in range(1, EPOCHS + 1): train_loss, train_acc = train(model, device, train_loader, optimizer, epoch) test_loss, test_acc = test(model, device, test_loader) # save train & test loss & accuracy train_loss_list.append(train_loss) train_acc_list.append(train_acc) test_loss_list.append(test_loss) test_acc_list.append(test_acc) end_time = time.time() plot_loss_acc(EPOCHS, train_loss_list, test_loss_list, 'CNN', 'with BN', loss=True) plot_loss_acc(EPOCHS, train_acc_list, test_acc_list, 'CNN', 'with BN', loss=False) print('Time per epoch = {:.2f}s'.format((end_time - start_time) / EPOCHS)) # with BN: 14.90s per epoch # without BN: 12.92s per epoch
train_acc_list.append(train_acc) LOG_INFO('Testing @ %d epoch...' % (epoch)) test_loss, test_acc, visual = test_net(model, loss, test_data, test_label, config['batch_size'], epoch, config['layer_vis']) # save loss & accuracy data for test epoch test_loss_list.append(test_loss) test_acc_list.append(test_acc) # plot visual during testing of the last epoch if epoch == (config['max_epoch'] - 1): plot_visual = visual[0:25, :, :] end_time = time.time() plot_loss_acc(config['max_epoch'], train_loss_list, test_loss_list, model.name, loss.name, loss=True) plot_loss_acc(config['max_epoch'], train_acc_list, test_acc_list, model.name, loss.name, loss=False) plot_vis(plot_visual, model.name, loss.name, config['layer_vis']) print('Time per epoch = ' + str((end_time - start_time) / config['max_epoch']) + 's')
def run_training(nb_epoch, model_filename, plot_path): """ Training the Fashion MNIST model :param nb_epoch: :return: """ # =================================================================== # Create the model # =================================================================== model = Sequential() model.add( Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=[28, 28, 1])) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(3, 3))) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(3, 3))) model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax')) # =================================================================== # Compile the model # =================================================================== model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) # =================================================================== # Train the model # =================================================================== print("[i] training the model...") history = model.fit_generator(data_generator(), steps_per_epoch=60000 // 32, nb_epoch=nb_epoch, validation_data=None) # =================================================================== # Save the best model # =================================================================== if not os.path.exists("models/"): print("[i] creating the [models/] directory...") os.makedirs("models/") else: print("[i] the [models/] directory already exists!") print("[i] saving the model...") model.save_weights('./models/' + model_filename) json_string = model.to_json() with open('./models/config.json', 'w') as f: f.write(json_string) # =================================================================== # Plot the training loss and accuracy # =================================================================== plot_loss_acc(history, nb_epoch, plot_path) print("********************* Done **********************")