# Compute quantities required for featurewise normalization # (std, mean, and principal components if ZCA whitening is applied). datagen.fit(X_train) # Fit the model on the batches generated by datagen.flow(). model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size, shuffle=True), steps_per_epoch=X_train.shape[0] // batch_size, validation_data=(X_test, Y_test), epochs=nb_epoch, verbose=1, callbacks=[lr_finder]) # from plot we see, the model isnt impacted by the weight_decay very much at all # so we can use any of them. for momentum in MOMENTUMS: directory = 'weights/momentum/momentum-%s/' % str(momentum) losses, lrs = lr_finder.restore_schedule_from_dir(directory, 10, 5) plt.plot(lrs, losses, label='momentum=%0.2f' % momentum) plt.title("Momentum") plt.xlabel("Learning rate") plt.ylabel("Validation Loss") plt.legend() plt.show()
def main(data_path=args['data_path'], train_from=train_from): train_gen, test_gen = create_data_generator(data_path) valid_x, valid_y = create_valid_data() MOMENTUMS = [0.9, 0.95, 0.99] for momentum in MOMENTUMS: K.clear_session() # Learning rate range obtained from `find_lr_schedule.py` # NOTE : Minimum is 10x smaller than the max found above ! # NOTE : It is preferable to use the validation data here to get a correct value lr_finder = LRFinder(NUM_SAMPLES, BS, minimum_lr=0.0001, maximum_lr=0.001, validation_data=(valid_x, valid_y), validation_sample_rate=5, lr_scale='linear', save_dir='weights/momentum/momentum-%s/' % str(momentum), verbose=True) if train_from == 'trained_weights': model = load_model_from_trained_weights( imagedims=IMAGE_DIMS, nb_classes=len(train_gen.class_indices), weights=args['weight_path'], freeze_until=freeze_until) elif train_from == 'trained_model': model = load_model_from_trained_model() else: model = load_models(imagedims=IMAGE_DIMS, nb_classes=len(train_gen.class_indices)) print('[INFO] compiling model...') model.compile(loss="categorical_crossentropy", optimizer=OPT, metrics=["accuracy"]) # set the weight_decay here ! # lr doesnt matter as it will be over written by the callback optimizer = SGD(lr=0.001, momentum=momentum, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) callbacks = [lr_finder] H = model.fit_generator( train_gen, validation_data=(valid_x, valid_y), epochs=EPOCHS, #steps_per_epoch=209, callbacks=callbacks, verbose=1) for momentum in MOMENTUMS: directory = 'weights/momentum/momentum-%s/' % str(momentum) losses, lrs = LRFinder.restore_schedule_from_dir(directory, 10, 5) plt.plot(lrs, losses, label='momentum=%0.2f' % momentum) plt.title("Momentum") plt.xlabel("Learning rate") plt.ylabel("Validation Loss") plt.legend() plt.show()