def train_vgg_model(weight_save, checkpoints_path, epochs=n_epochs): model = vgg_unet(n_classes=n_classes, input_height=input_height, input_width=input_width) # model.load_weights(os.path.join(checkpt_paths, '.38')) # Set a cyclical learning rate as a callback. The learning rate varies between 0.005 and 0.0001 # A triangle shape of cycle is used cyclic_lr = CyclicLR(base_lr=0.001, max_lr=0.01, step_size=1024, mode='triangular') print_lr = PrintLR() model.train(train_images=os.path.join(AUGMENTED_IMAGES_DIRECTORY, 'img'), train_annotations=os.path.join(AUGMENTED_LABEL_DIRECTORY, 'img'), auto_resume_checkpoint=True, checkpoints_path=checkpoints_path, verify_dataset=False, epochs=epochs, more_callbacks=[cyclic_lr, print_lr]) model.save_weights(os.path.join(MODELS_DIRECTORY, weight_save)) return model
def update_model(self, stage_params): dataset = self.memory.recall(shuffle=True, n_sample=stage_params["training_samples"]) dataset_size = dataset["Boards"].shape[0] train_select = np.random.choice(a=[False, True], size=dataset_size, p=[0.1, 0.9]) # Protect against the case when the val split means there is no val # data if train_select.sum() == dataset_size: train_select[0] = False self.logger.debug("Validation size is 0, setting 1 example.") validation_select = ~train_select train_boards = dataset["Boards"][train_select] train_policies = dataset["Policies"][train_select] train_values = dataset["Values"][train_select] validation_boards = dataset["Boards"][validation_select] validation_policies = dataset["Policies"][validation_select] validation_values = dataset["Values"][validation_select] validation_data = ( validation_boards, { "value": validation_values, "policy": validation_policies }, ) patience = stage_params["update_epochs"] // 5 early_stopping = tf.keras.callbacks.EarlyStopping( patience=patience, restore_best_weights=True) clr = CyclicLR( base_lr=stage_params["learning_rate"], max_lr=stage_params["learning_rate"] / 3, step_size=4 * len(train_boards) // 64, mode="triangular", ) train_history = self.model.fit( train_boards, { "value": train_values, "policy": train_policies }, validation_data=validation_data, batch_size=512, epochs=stage_params["update_epochs"], verbose=1, callbacks=[clr, early_stopping], ) return train_history
model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation("relu")) model.add(Dropout(0.5)) model.add(Dense(num_classes)) model.add(Activation("softmax")) # initiate RMSprop optimizer opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6) # initiate CyclicLR LR scheduler clr = CyclicLR(base_lr=0.0001, max_lr=0.0005, step_size=2000, mode="triangular") # Let's train the model using RMSprop model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) x_train = x_train.astype("float32") x_test = x_test.astype("float32") x_train /= 255 x_test /= 255 if not data_augmentation: print("Not using data augmentation.") model.fit(
def regressor_training_main(folders, val_folders, load_model, model_name, time, epochs, batch_size, opt, learning_rate, lropf, sd, es, feature, workers, test_dirs, intensity_cut, leakage, tb, gpu_fraction=0.5, emin=-100, emax=100, class_model='', clr=False, train_indexes=None, valid_indexes=None, clr_values=[5e-5, 5e-3, 4]): ################################### #TensorFlow wizardry for GPU dynamic memory allocation if 0 < gpu_fraction < 1: config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a fraction of the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction # Create a session with the above options specified. K.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # remove semaphore warnings os.environ["PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning" # avoid validation deadlock problem mp.set_start_method('spawn', force=True) # hard coded parameters shuffle = True channels = 1 if time: channels = 2 # early stopping md_es = 0.1 # min delta p_es = 50 # patience # cycle learning rate CLR base_lr = clr_values[0] max_lr = clr_values[1] step_size = clr_values[2] # sgd lr = 0.01 # lr decay = 1e-4 # decay momentum = 0.9 # momentum nesterov = True # adam a_lr = learning_rate a_beta_1 = 0.9 a_beta_2 = 0.999 a_epsilon = None a_decay = 0 amsgrad = True # adabound ab_lr = 1e-03 ab_final_lr = 0.1 ab_gamma = 1e-03 ab_weight_decay = 0 amsbound = False # rmsprop r_lr = 0.01 r_rho = 0.9 r_epsilon = None r_decay = 0.0 # reduce lr on plateau f_lrop = 0.1 # factor p_lrop = 15 # patience md_lrop = 0.005 # min delta cd_lrop = 5 # cool down mlr_lrop = a_lr / 100 # min lr # cuts # intensity_cut = 50 leakage2_intensity_cut = leakage training_files = get_all_files(folders) validation_files = get_all_files(val_folders) # create a folder to keep model & results now = datetime.datetime.now() root_dir = now.strftime(model_name + '_' + feature + '_' + '%Y-%m-%d_%H-%M') mkdir(root_dir) models_dir = join(root_dir, "models") mkdir(models_dir) # generators print('Building training generator...') training_generator = LSTGenerator( training_files, batch_size=batch_size, arrival_time=time, feature=feature, shuffle=shuffle, intensity=intensity_cut, leakage2_intensity=leakage2_intensity_cut, load_indexes=train_indexes) if train_indexes is None: train_idxs = training_generator.get_all_info() train_idxs.to_pickle(join(root_dir, "train_indexes.pkl")) if len(val_folders) > 0: print('Building validation generator...') validation_generator = LSTGenerator( validation_files, batch_size=batch_size, arrival_time=time, feature=feature, shuffle=False, intensity=intensity_cut, leakage2_intensity=leakage2_intensity_cut, load_indexes=valid_indexes) if valid_indexes is None: valid_idxs = validation_generator.get_all_info() valid_idxs.to_pickle(join(root_dir, "valid_indexes.pkl")) # class_weight = {0: 1., 1: train_protons/train_gammas} # print(class_weight) # get image size (rows and columns) img_rows = training_generator.img_rows img_cols = training_generator.img_cols hype_print = '\n' + '======================================HYPERPARAMETERS======================================' hype_print += '\n' + 'Image rows: ' + str( img_rows) + ' Image cols: ' + str(img_cols) hype_print += '\n' + 'Folders:' + str(folders) hype_print += '\n' + 'Model: ' + str(model_name) hype_print += '\n' + 'Use arrival time: ' + str(time) hype_print += '\n' + 'Epochs:' + str(epochs) hype_print += '\n' + 'Batch size: ' + str(batch_size) hype_print += '\n' + 'Optimizer: ' + str(opt) hype_print += '\n' + 'Feature: ' + str(feature) hype_print += '\n' + 'Validation: ' + str(val_folders) hype_print += '\n' + 'Test dirs: ' + str(test_dirs) hype_print += '\n' + 'intensity_cut: ' + str(intensity_cut) hype_print += '\n' + 'leakage2_intensity_cut: ' + str( leakage2_intensity_cut) if clr: hype_print += '\n' + '--- Cycle Learning Rate ---' hype_print += '\n' + 'Base LR: ' + str(base_lr) hype_print += '\n' + 'Max LR: ' + str(max_lr) hype_print += '\n' + 'Step size: ' + str(step_size) + ' (' + str( step_size * len(training_generator)) + ')' if es: hype_print += '\n' + '--- Early stopping ---' hype_print += '\n' + 'Min delta: ' + str(md_es) hype_print += '\n' + 'Patience: ' + str(p_es) hype_print += '\n' + '----------------------' if opt == 'sgd': hype_print += '\n' + '--- SGD ---' hype_print += '\n' + 'Learning rate:' + str(lr) hype_print += '\n' + 'Decay: ' + str(decay) hype_print += '\n' + 'Momentum: ' + str(momentum) hype_print += '\n' + 'Nesterov: ' + str(nesterov) hype_print += '\n' + '-----------' elif opt == 'adam': hype_print += '\n' + '--- ADAM ---' hype_print += '\n' + 'lr: ' + str(a_lr) hype_print += '\n' + 'beta_1: ' + str(a_beta_1) hype_print += '\n' + 'beta_2: ' + str(a_beta_2) hype_print += '\n' + 'epsilon: ' + str(a_epsilon) hype_print += '\n' + 'decay: ' + str(a_decay) hype_print += '\n' + 'Amsgrad: ' + str(amsgrad) hype_print += '\n' + '------------' elif opt == 'rmsprop': hype_print += '\n' + '--- RMSprop ---' hype_print += '\n' + 'lr: ' + str(r_lr) hype_print += '\n' + 'rho: ' + str(r_rho) hype_print += '\n' + 'epsilon: ' + str(r_epsilon) hype_print += '\n' + 'decay: ' + str(r_decay) hype_print += '\n' + '------------' if lropf: hype_print += '\n' + '--- Reduce lr on plateau ---' hype_print += '\n' + 'lr decrease factor: ' + str(f_lrop) hype_print += '\n' + 'Patience: ' + str(p_lrop) hype_print += '\n' + 'Min delta: ' + str(md_lrop) hype_print += '\n' + 'Cool down:' + str(cd_lrop) hype_print += '\n' + 'Min lr: ' + str(mlr_lrop) hype_print += '\n' + '----------------------------' if sd: hype_print += '\n' + '--- Step decay ---' hype_print += '\n' + 'Workers: ' + str(workers) hype_print += '\n' + 'Shuffle: ' + str(shuffle) hype_print += '\n' + 'Number of training batches: ' + str( len(training_generator)) if len(val_folders) > 0: hype_print += '\n' + 'Number of validation batches: ' + str( len(validation_generator)) outcomes = 1 # loss = 'mean_absolute_percentage_error' loss = 'mean_absolute_error' # loss = 'mean_squared_error' if feature == 'direction': outcomes = 2 # loss = 'mean_absolute_error' # loss = 'mean_squared_error' # keras.backend.set_image_data_format('channels_first') if load_model: model = keras.models.load_model(model_name) model_name = Path(model_name).name else: model, hype_print = regressor_selector(model_name, hype_print, channels, img_rows, img_cols, outcomes) hype_print += '\n' + '=========================================================================================' # printing on screen hyperparameters print(hype_print) # writing hyperparameters on file f = open(root_dir + '/hyperparameters.txt', 'w') f.write(hype_print) f.close() model.summary() callbacks = [] if len(val_folders) > 0: checkpoint = ModelCheckpoint( filepath=models_dir + '/' + model_name + '_{epoch:02d}_{loss:.5f}_{val_loss:.5f}.h5', monitor='val_loss', save_best_only=False) else: checkpoint = ModelCheckpoint(filepath=models_dir + '/' + model_name + '_{epoch:02d}_{loss:.5f}.h5', monitor='loss', save_best_only=True) callbacks.append(checkpoint) # tensorboard = keras.callbacks.TensorBoard(log_dir=root_dir + "/logs", # histogram_freq=5, # batch_size=batch_size, # write_images=True, # update_freq=batch_size * 100) history = LossHistoryR() csv_callback = keras.callbacks.CSVLogger(root_dir + '/epochs_log.csv', separator=',', append=False) callbacks.append(history) callbacks.append(csv_callback) # callbacks.append(tensorboard) # sgd optimizer = None if opt == 'sgd': sgd = optimizers.SGD(lr=lr, decay=decay, momentum=momentum, nesterov=nesterov) optimizer = sgd elif opt == 'adam': adam = optimizers.Adam(lr=a_lr, beta_1=a_beta_1, beta_2=a_beta_2, epsilon=a_epsilon, decay=a_decay, amsgrad=amsgrad) optimizer = adam # elif opt == 'adabound': # adabound = AdaBound(lr=ab_lr, final_lr=ab_final_lr, gamma=ab_gamma, weight_decay=ab_weight_decay, # amsbound=False) # optimizer = adabound elif opt == 'rmsprop': rmsprop = optimizers.RMSprop(lr=r_lr, rho=r_rho, epsilon=r_epsilon, decay=r_decay) optimizer = rmsprop # reduce lr on plateau if lropf: lrop = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=f_lrop, patience=p_lrop, verbose=1, mode='auto', min_delta=md_lrop, cooldown=cd_lrop, min_lr=mlr_lrop) callbacks.append(lrop) if sd: # learning rate schedule def step_decay(epoch): current = K.eval(model.optimizer.lr) lrate = current if epoch == 99: lrate = current / 10 print('Reduced learning rate by a factor 10') return lrate stepd = LearningRateScheduler(step_decay) callbacks.append(stepd) if es: # early stopping early_stopping = EarlyStopping(monitor='val_loss', min_delta=md_es, patience=p_es, verbose=1, mode='max') callbacks.append(early_stopping) if tb: tb_path = os.path.join(root_dir, 'tb') if not os.path.exists(tb_path): os.mkdir(tb_path) #tb_path = os.path.join(tb_path, root_dir) # os.mkdir(tb_path) tensorboard = TensorBoard(log_dir=tb_path) callbacks.append(tensorboard) ''' findlr=False if findlr: lr_callback = LRFinder(len(training_generator)*batch_size, batch_size, 1e-05, 1e01, # validation_data=(X_val, Y_val), lr_scale='exp', save_dir=join(root_dir,'clr')) callbacks.append(lr_callback) oclr=True #CORREGGI STA MERDA OPPURE CANCELLA if oclr: lr_manager = OneCycleLR( max_lr=0.001, maximum_momentum=0.9, minimum_momentum=None, verbose=True ) callbacks.append(lr_manager) oclr=True if oclr: ocp = ktools.one_cycle.OneCycle( lr_range=(1e-5, 1e-3), momentum_range=(0.95, 0.85), reset_on_train_begin=True, record_frq=10 ) callbacks.append(ocp) ''' if clr: cyclelr = CyclicLR( base_lr=base_lr, max_lr=max_lr, step_size=step_size * len(training_generator), #mode='exp_range' # uncomment to activate exp mode, instead of traingular ) callbacks.append(cyclelr) model.compile(optimizer=optimizer, loss=loss) if len(val_folders) > 0: model.fit(x=training_generator, validation_data=validation_generator, steps_per_epoch=len(training_generator), validation_steps=len(validation_generator), epochs=epochs, verbose=1, max_queue_size=10, use_multiprocessing=True, workers=workers, shuffle=False, callbacks=callbacks) else: model.fit(x=training_generator, steps_per_epoch=len(training_generator), epochs=epochs, verbose=1, max_queue_size=10, use_multiprocessing=True, workers=workers, shuffle=False, callbacks=callbacks) # mp.set_start_method('fork') # save results train_history = root_dir + '/train-history' with open(train_history, 'wb') as file_pi: pickle.dump(history.dic, file_pi) # post training operations # training plots train_plots(train_history, False) if len(test_dirs) > 0: if len(val_folders) > 0: # get the best model on validation val_loss = history.dic['val_losses'] m = val_loss.index( min(val_loss)) # get the index with the highest accuracy model_checkpoints = [ join(root_dir, f) for f in listdir(root_dir) if (isfile(join(root_dir, f)) and f.startswith( model_name + '_' + '{:02d}'.format(m + 1))) ] best = model_checkpoints[0] print('Best checkpoint: ', best) else: # get the best model on validation acc = history.dic['losses'] m = acc.index(min(acc)) # get the index with the highest accuracy model_checkpoints = [ join(root_dir, f) for f in listdir(root_dir) if (isfile(join(root_dir, f)) and f.startswith( model_name + '_' + '{:02d}'.format(m + 1))) ] best = model_checkpoints[0] print('Best checkpoint: ', best) # test plots & results if test data is provided if len(test_dirs) > 0: pkl = tester(test_dirs, best, batch_size, time, feature, workers) test_plots(pkl, feature)
model = Model(inputs=inp, outputs=pred) model.compile(loss = 'sparse_categorical_crossentropy' , metrics = ['accuracy'] , optimizer = 'nadam') return model K.clear_session() model = gen_model(n_classes) #model = gen_model(58) model.summary() batch_size = 512 epochs = 5 step_size = int(int(len(X_train)/batch_size)*epochs/2) cb = [ CyclicLR(5e-4 , 2e-3 , step_size) ] model.fit(X_train, y_train, validation_data = (X_val, y_val), epochs = epochs, batch_size= batch_size, verbose = 1, shuffle = True, callbacks = cb) preds = model.predict(test_text,verbose = 1).argmax(axis = 1)
def classifier_training_main( folders, val_folders, model_name, time, epochs, batch_size, opt, learn_rate, lropf=False, sd=False, es=False, clr=False, workers=1, test_dirs='', load_model=False, tb=False, intensity_cut=None, leakage=0.2, gpu_fraction=1, train_indexes=None, valid_indexes=None, clr_values=[5e-5, 5e-3, 4]): if 0 >= gpu_fraction or gpu_fraction > 1: pass ################################### # TensorFlow wizardry for GPU dynamic memory allocation else: config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Only allow a fraction of the GPU memory to be allocated config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction # Create a session with the above options specified. K.tensorflow_backend.set_session(tf.Session(config=config)) ################################### # remove semaphore warnings os.environ["PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning" # avoid validation deadlock problem mp.set_start_method('spawn', force=True) # hard coded parameters shuffle = True channels = 1 if time: channels = 2 # early stopping md_es = 0.01 # min delta p_es = 25 # patience # cycle learning rate CLR base_lr = clr_values[0] max_lr = clr_values[1] step_size = clr_values[2] # sgd lr = 0.01 # lr decay = 1e-4 # decay momentum = 0.9 # momentum nesterov = True # adam # default a_lr should be 0.001 a_lr = learn_rate a_beta_1 = 0.9 a_beta_2 = 0.999 a_epsilon = None a_decay = 0 amsgrad = True # adabound ab_lr = 1e-03 ab_final_lr = 0.1 ab_gamma = 1e-03 ab_weight_decay = 0 amsbound = True # reduce lr on plateau f_lrop = 0.1 # factor p_lrop = 15 # patience md_lrop = 0.005 # min delta cd_lrop = 5 # cool down mlr_lrop = a_lr / 100 # min lr # cuts #intensity_cut = 500 #leakage2_intensity_cut = 0.2 training_files = get_all_files(folders) validation_files = get_all_files(val_folders) # generators print('Building training generator...') feature = 'gammaness' # hardcoded by now ''' training_generator = DataGeneratorC(training_files, batch_size=batch_size, arrival_time=time, shuffle=shuffle, intensity=intensity_cut) train_idxs = training_generator.get_indexes() train_gammas = np.unique(train_idxs[:, 2], return_counts=True)[1][1] train_protons = np.unique(train_idxs[:, 2], return_counts=True)[1][0] ''' training_generator = LSTGenerator(training_files, batch_size=batch_size, arrival_time=time, feature=feature, shuffle=shuffle, intensity=intensity_cut, leakage2_intensity=leakage, load_indexes=train_indexes) # get image size (rows and columns) img_rows = training_generator.img_rows img_cols = training_generator.img_cols print("IMG rows: {}, cols: {}".format(img_rows, img_cols)) # create a folder to keep model & results now = datetime.datetime.now() root_dir = now.strftime(model_name + '_' + '%Y-%m-%d_%H-%M') mkdir(root_dir) models_dir = join(root_dir, "models") mkdir(models_dir) # save data info train_idxs = training_generator.get_all_info() train_gammas = np.unique(train_idxs['class'], return_counts=True)[1][1] train_protons = np.unique(train_idxs['class'], return_counts=True)[1][0] train_gamma_frac = training_generator.gamma_fraction() if train_indexes is None: train_idxs.to_pickle(join(root_dir, "train_indexes.pkl")) if len(val_folders) > 0: print('Building validation generator...') validation_generator = LSTGenerator(validation_files, batch_size=batch_size, arrival_time=time, feature=feature, shuffle=False, intensity=intensity_cut, leakage2_intensity=leakage, load_indexes=valid_indexes ) valid_idxs = validation_generator.get_all_info() valid_gammas = np.unique(valid_idxs['class'], return_counts=True)[1][1] valid_protons = np.unique(valid_idxs['class'], return_counts=True)[1][0] valid_gamma_frac = validation_generator.gamma_fraction() if valid_indexes is None: valid_idxs.to_pickle(join(root_dir, "valid_indexes.pkl")) # class_weight = {0: 1., 1: train_protons/train_gammas} # print(class_weight) hype_print = '\n' + '======================================HYPERPARAMETERS======================================' hype_print += '\n' + 'Image rows: ' + str(img_rows) + ' Image cols: ' + str(img_cols) hype_print += '\n' + 'Folders:' + str(folders) hype_print += '\n' + 'Model: ' + str(model_name) hype_print += '\n' + 'Use arrival time: ' + str(time) hype_print += '\n' + 'Epochs:' + str(epochs) hype_print += '\n' + 'Batch size: ' + str(batch_size) hype_print += '\n' + 'Optimizer: ' + str(opt) hype_print += '\n' + 'Validation: ' + str(val_folders) hype_print += '\n' + 'Test dirs: ' + str(test_dirs) hype_print += '\n' + 'intensity_cut: ' + str(intensity_cut) hype_print += '\n' + 'leakage2_intensity_cut: ' + str(leakage) if clr: hype_print += '\n' + '--- Cycle Learning Rate ---' hype_print += '\n' + 'Base LR: ' + str(base_lr) hype_print += '\n' + 'Max LR: ' + str(max_lr) hype_print += '\n' + 'Step size: ' + str(step_size) + ' (' + str(step_size*len(training_generator)) + ')' if es: hype_print += '\n' + '--- Early stopping ---' hype_print += '\n' + 'Min delta: ' + str(md_es) hype_print += '\n' + 'Patience: ' + str(p_es) hype_print += '\n' + '----------------------' if opt == 'sgd': hype_print += '\n' + '--- SGD ---' hype_print += '\n' + 'Learning rate:' + str(lr) hype_print += '\n' + 'Decay: ' + str(decay) hype_print += '\n' + 'Momentum: ' + str(momentum) hype_print += '\n' + 'Nesterov: ' + str(nesterov) hype_print += '\n' + '-----------' elif opt == 'adam': hype_print += '\n' + '--- ADAM ---' hype_print += '\n' + 'lr: ' + str(a_lr) hype_print += '\n' + 'beta_1: ' + str(a_beta_1) hype_print += '\n' + 'beta_2: ' + str(a_beta_2) hype_print += '\n' + 'epsilon: ' + str(a_epsilon) hype_print += '\n' + 'decay: ' + str(a_decay) hype_print += '\n' + 'Amsgrad: ' + str(amsgrad) hype_print += '\n' + '------------' if lropf: hype_print += '\n' + '--- Reduce lr on plateau ---' hype_print += '\n' + 'lr decrease factor: ' + str(f_lrop) hype_print += '\n' + 'Patience: ' + str(p_lrop) hype_print += '\n' + 'Min delta: ' + str(md_lrop) hype_print += '\n' + 'Cool down:' + str(cd_lrop) hype_print += '\n' + 'Min lr: ' + str(mlr_lrop) hype_print += '\n' + '----------------------------' if sd: hype_print += '\n' + '--- Step decay ---' hype_print += '\n' + 'Workers: ' + str(workers) hype_print += '\n' + 'Shuffle: ' + str(shuffle) hype_print += '\n' + 'Number of training batches: ' + str(len(training_generator)) hype_print += '\n' + 'Number of training gammas: ' + str(train_gammas) hype_print += '\n' + 'Number of training protons: ' + str(train_protons) hype_print += '\n' + 'Fraction of gamma in training set: ' + str(train_gamma_frac) if len(val_folders) > 0: hype_print += '\n' + 'Number of validation batches: ' + str(len(validation_generator)) hype_print += '\n' + 'Number of validation gammas: ' + str(valid_gammas) hype_print += '\n' + 'Number of validation protons: ' + str(valid_protons) hype_print += '\n' + 'Fraction of gamma in validation set: ' + str(valid_gamma_frac) # keras.backend.set_image_data_format('channels_first') if load_model: model = keras.models.load_model(model_name) model_name = Path(model_name).name else: model, hype_print = select_classifier(model_name, hype_print, channels, img_rows, img_cols) #model = load_model('/home/pgrespan/nick_models/ResNetFSE_49_0.82375_0.80292.h5') hype_print += '\n' + '=========================================================================================' # printing on screen hyperparameters print(hype_print) # writing hyperparameters on file f = open(root_dir + '/hyperparameters.txt', 'w') f.write(hype_print) f.close() model.summary() callbacks = [] if len(val_folders) > 0: checkpoint = ModelCheckpoint( filepath=models_dir + '/' + model_name + '_{epoch:02d}_{acc:.5f}_{val_acc:.5f}.h5', monitor='val_acc', save_best_only=False) else: checkpoint = ModelCheckpoint( filepath=models_dir + '/' + model_name + '_{epoch:02d}_{acc:.5f}.h5', monitor='acc', save_best_only=True) callbacks.append(checkpoint) # tensorboard = keras.callbacks.TensorBoard(log_dir=root_dir + "/logs", # histogram_freq=5, # batch_size=batch_size, # write_images=True, # update_freq=batch_size * 100) history = LossHistoryC() csv_callback = keras.callbacks.CSVLogger(root_dir + '/epochs_log.csv', separator=',', append=False) callbacks.append(history) callbacks.append(csv_callback) # callbacks.append(tensorboard) # sgd optimizer = None if opt == 'sgd': sgd = optimizers.SGD(lr=lr, decay=decay, momentum=momentum, nesterov=nesterov) optimizer = sgd elif opt == 'adam': adam = optimizers.Adam(lr=a_lr, beta_1=a_beta_1, beta_2=a_beta_2, epsilon=a_epsilon, decay=a_decay, amsgrad=amsgrad) optimizer = adam ''' elif opt == 'adabound': adabound = AdaBound(lr=ab_lr, final_lr=ab_final_lr, gamma=ab_gamma, weight_decay=ab_weight_decay, amsbound=False) optimizer = adabound ''' # reduce lr on plateau if lropf: lrop = keras.callbacks.ReduceLROnPlateau(monitor='val_acc', factor=f_lrop, patience=p_lrop, verbose=1, mode='auto', min_delta=md_lrop, cooldown=cd_lrop, min_lr=mlr_lrop) callbacks.append(lrop) if sd: # learning rate schedule def step_decay(epoch): current = K.eval(model.optimizer.lr) lrate = current if epoch == 99: lrate = current / 10 print('Reduced learning rate by a factor 10') return lrate stepd = LearningRateScheduler(step_decay) callbacks.append(stepd) if es: # early stopping early_stopping = EarlyStopping(monitor='val_acc', min_delta=md_es, patience=p_es, verbose=1, mode='max') callbacks.append(early_stopping) if tb: tb_path = os.path.join(root_dir, 'tb') if not os.path.exists(tb_path): os.mkdir(tb_path) #tb_path = os.path.join(tb_path, root_dir) # os.mkdir(tb_path) tensorboard = TensorBoard(log_dir=tb_path) callbacks.append(tensorboard) if clr: cyclelr = CyclicLR( base_lr=base_lr, max_lr=max_lr, step_size=step_size*len(training_generator) ) callbacks.append(cyclelr) lrfinder=False if lrfinder: lr_callback = LRFinder(len(training_generator)*batch_size, batch_size, 1e-05, 1e01, # validation_data=(X_val, Y_val), lr_scale='exp', save_dir=join(root_dir,'clr')) callbacks.append(lr_callback) model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy']) if len(val_folders) > 0: model.fit( x=training_generator, validation_data=validation_generator, steps_per_epoch=len(training_generator), validation_steps=len(validation_generator), epochs=epochs, verbose=1, max_queue_size=10, use_multiprocessing=True, workers=workers, shuffle=False, callbacks=callbacks ) else: model.fit( x=training_generator, steps_per_epoch=len(training_generator), epochs=epochs, verbose=1, max_queue_size=10, use_multiprocessing=True, workers=workers, shuffle=False, callbacks=callbacks ) # save results train_history = root_dir + '/train-history' with open(train_history, 'wb') as file_pi: pickle.dump(history.dic, file_pi) # post training operations # training plots train_plots(train_history, False) if len(test_dirs) > 0: if len(val_folders) > 0: # get the best model on validation val_acc = history.dic['val_accuracy'] m = val_acc.index(max(val_acc)) # get the index with the highest accuracy model_checkpoints = [join(root_dir, f) for f in listdir(root_dir) if (isfile(join(root_dir, f)) and f.startswith( model_name + '_' + '{:02d}'.format(m + 1)))] best = model_checkpoints[0] print('Best checkpoint: ', best) else: # get the best model acc = history.dic['accuracy'] m = acc.index(max(acc)) # get the index with the highest accuracy model_checkpoints = [join(root_dir, f) for f in listdir(root_dir) if (isfile(join(root_dir, f)) and f.startswith( model_name + '_' + '{:02d}'.format(m + 1)))] best = model_checkpoints[0] print('Best checkpoint: ', best) # test plots & results if test data is provided if len(test_dirs) > 0: csv = tester(test_dirs, best, batch_size, time, workers) test_plots(csv)