def callbacks(cp, tb, logs): # Callbacks: Save the model. directory1 = os.path.join('out', 'checkpoints', cp) if not os.path.exists(directory1): os.makedirs(directory1) checkpointer = ModelCheckpoint(filepath=os.path.join( directory1, '{epoch:03d}.hdf5'), monitor='accuracy', verbos=1, save_best_only=True, save_weights_only=True, mode='max') # Callbacks: TensorBoard directory2 = os.path.join('out', 'tensorboard', tb) if not os.path.exists(directory2): os.makedirs(directory2) tensorboard = TensorBoard(log_dir=os.path.join(directory2)) # Callbacks: Early Stopper early_stopper = EarlyStopping(monitor='loss', patience=100) # Callbacks: Save Results. directory3 = os.path.join('out', 'results', logs) if not os.path.exists(directory3): os.makedirs(directory3) csv_logger = CSVLogger( os.path.join(directory3, 'training-' + 'logs' + '.csv')) # Learning rate schedule. lr_schedule = LearningRateScheduler(fixed_schedule, verbose=0) cb = [ tensorboard, early_stopper, csv_logger, checkpointer, lr_schedule, CustomCallback() ] return cb
def train(lambd, sigma, n_centers, trial): K.clear_session() (X_train, y_train), (X_test, y_test) = inbalanced_cifar(200) model = create_models(sigma, n_centers) model.compile("adam", affinity_loss(lambd), [acc]) tf.logging.set_verbosity(tf.logging.FATAL) # ログを埋めないようにする tpu_grpc_url = "grpc://"+os.environ["COLAB_TPU_ADDR"] tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(tpu_grpc_url) strategy = keras_support.TPUDistributionStrategy(tpu_cluster_resolver) model = tf.contrib.tpu.keras_to_tpu_model(model, strategy=strategy) scheduler = LearningRateScheduler(step_decay) f1 = F1Callback(model, X_test, y_test, trial) history = model.fit(X_train, y_train, callbacks=[scheduler, f1], batch_size=640, epochs=100, verbose=0).history max_f1 = max(f1.f1_log) print("lambda:{lambd:.04}, sigma:{sigma:.04} n_centers:{n_centers} / f1 = {max_f1:.04}") return max_f1
def initialize_callbacks(self): checkpoint_path = os.path.join( self.output_dir, self.run_uid + "_" + self.model_string + "-{epoch:04d}-{val_loss:.4f}.h5", ) checkpoint = ModelCheckpoint( checkpoint_path, monitor="val_loss", verbose=1, save_best_only=True, mode="min", period=self.period_save, ) # Add on epoch_end callback epo_end = OnEpochEnd([self.local_generator.on_epoch_end]) if self.apply_learning_decay == 1: step_decay_callback = create_decay_callback( self.initial_learning_rate, self.epochs_drop) lrate = LearningRateScheduler(step_decay_callback) callbacks_list = [checkpoint, lrate] else: callbacks_list = [checkpoint] if not (self.tensorboard_path == ""): board_callback = tensorflow.keras.callbacks.TensorBoard( log_dir=os.path.join(self.tensorboard_path, self.model_string), histogram_freq=1, ) callbacks_list.append(board_callback) if not (tensorflow.__version__ == '2.3.0'): callbacks_list.append(epo_end) self.callbacks_list = callbacks_list
def train_dnn(epochs=10, num_rounds=5, num_layers=2): model = make_dnn(num_layers=num_layers) model.compile(optimizer='adam', loss='mse', metrics=['acc']) model.summary() x_train, y_train = samples.make_train_data(10**7, num_rounds) x_test, y_test = samples.make_train_data(10**6, num_rounds) lr = LearningRateScheduler(cyclic_lr(9, 0.0001, 0.002)) check = make_checkpoint('best_dnn_r' + str(num_rounds) + "_l" + str(num_layers) + ".h5") h = model.fit(x_train, y_train, epochs=epochs, batch_size=5000, validation_data=(x_test, y_test), callbacks=[lr, check]) print("Best validaation accuracy: ", np.max(h.history['val_acc'])) return model
def warmup(self, x_train, y_train, epochs=5, s_lr=1e-6, e_lr=0.001): """ Warmup for numerical stability x_train : training images y_train : training labels epochs : number of epochs for warmup s_lr : start warmup learning rate e_lr : end warmup learning rate """ print("*** Warmup (for numerical stability)") # Setup learning rate scheduler self.compile(optimizer=Adam(s_lr)) lrate = LearningRateScheduler(self.warmup_scheduler, verbose=1) self.w_epochs = epochs self.w_lr = e_lr - s_lr # Train the model self.model.fit(x_train, y_train, epochs=epochs, batch_size=32, verbose=1, callbacks=[lrate])
def warmup(self, x_train, y_train, epochs=5, batch_size=32, s_lr=1e-6, e_lr=0.001, loss='categorical_crossentropy', metrics=['acc']): """ Warmup for numerical stability x_train : training images y_train : training labels epochs : number of epochs for warmup batch_size: batch size s_lr : start warmup learning rate e_lr : end warmup learning rate loss : loss function metrics : training metrics to report """ print("*** Warmup (for numerical stability)") # Setup learning rate scheduler self.compile(optimizer=Adam(s_lr), loss=loss, metrics=metrics) lrate = LearningRateScheduler(self.warmup_scheduler, verbose=1) self.w_epochs = epochs self.w_lr = e_lr - s_lr # Train the model self.model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1, callbacks=[lrate])
def training(self, x_train, y_train, epochs=10, batch_size=32, lr=0.001, decay=1e-05): """ Full Training of the Model x_train : training images y_train : training labels epochs : number of epochs batch_size : size of batch lr : learning rate decay : learning rate decay """ # Check for hidden dropout layer in classifier for layer in self.model.layers: if isinstance(layer, Dropout): self.hidden_dropout = layer break self.t_decay = decay self.compile(optimizer=Adam(lr=lr, decay=decay)) lrate = LearningRateScheduler(self.training_scheduler, verbose=1) self.model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, validation_split=0.1, verbose=1, callbacks=[lrate])
def callbacks_(self): '''callbacks''' lr_scheduler = LearningRateScheduler(self.lr_schedule) checkpoint = ModelCheckpoint(filepath = self.params.model_directory + "/weights.hdf5", monitor = 'val_loss', save_best_only = True, verbose = 1, save_weights_only = True) tb = TensorBoard(log_dir = self.params.model_directory + "/tensorboard", histogram_freq = 0, write_graph = True, write_images = True, embeddings_layer_names = None, embeddings_metadata = None) csv_logger = CSVLogger(filename = self.params.model_directory + '/history.csv', append = True, separator = ",") return [lr_scheduler, checkpoint, tb, csv_logger]
def build_model(n_kernels, kernel_size, stride, n_dense): model = tf.keras.models.Sequential() model.add(layers.Convolution2D(filters=n_kernels,kernel_size=(kernel_size,kernel_size),activation='relu',input_shape=(16,16,1))) model.add(layers.BatchNormalization()) model.add(layers.MaxPooling2D(strides=(stride,stride))) model.add(layers.Dropout(rate=0.25)) model.add(layers.Flatten()) model.add(layers.Dense(n_dense,activation='relu')) model.add(layers.Dropout(rate=0.5)) model.add(layers.Dense(10,activation='softmax')) adamOptimizer= tf.keras.optimizers.Adam(lr=0.0001) model.compile(optimizer=adamOptimizer,loss='categorical_crossentropy') annealer = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x) history = model.fit(x_trn,y_trn,epochs=2,batch_size =16, verbose=2, validation_data=(x_val, y_val),callbacks=[annealer]) tstError=model.evaluate(x_tst,y_tst) trnError=model.evaluate(x_trn,y_trn) noOfParams=model.count_params() return (tstError,trnError,noOfParams)
def _make_callback(args, save_models): """ Generate the callback. Parameters ---------- args: dic A dictionary containing all of the input parameters. save_models: str Path to the output directory for the models. Returns ------- callbacks: obj List of callback objects. """ m_name = str(args['output_name']) + '_{epoch:03d}.h5' filepath = os.path.join(save_models, m_name) early_stopping_monitor = EarlyStopping(monitor=args['monitor'], patience=args['patience']) checkpoint = ModelCheckpoint(filepath=filepath, monitor=args['monitor'], mode='auto', verbose=1, save_best_only=True) lr_scheduler = LearningRateScheduler(_lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=args['patience'] - 2, min_lr=0.5e-6) callbacks = [checkpoint, lr_reducer, lr_scheduler, early_stopping_monitor] return callbacks
def train(weights_path, epochs, batch_size, initial_epoch): """Trains a model.""" print ('loading data...') # Loads or creates training data. input_shape, train, valid, train_targets, valid_targets = get_train_data() print ('getting model...') # Loads or creates model. model, checkpoint_path = get_model(input_shape, scale_factor=len(train)/batch_size, weights_path=weights_path) # Sets callbacks. checkpointer = ModelCheckpoint(checkpoint_path, verbose=1, save_weights_only=True, save_best_only=True) scheduler = LearningRateScheduler(schedule) print ('fitting model...') # Trains model. model.fit(train, train_targets, batch_size, epochs, initial_epoch=initial_epoch, callbacks=[checkpointer, scheduler], validation_data=(valid, valid_targets))
def get_callbacks(self, gen_bool, val_gen, val_y, train_gen=None, train_y=None): # callbacks when fitting early_stopping = OnAllValDataEarlyStopping( self.dataset.name, gen_bool, val_gen, val_y, train_gen, train_y, qsize=1000, workers=5, monitor=self.dataset.ea_metric, mode=self.dataset.ea_mode, # minimum change to qualify as an improvement min_delta=0, patience=self.patience_early_stopping, verbose=1, restore_best_weights=True) csv_logger = CSVLogger('temp.log') # streams epoch results to a csv if self.lr_scheduler['name'] == 'ReduceLROnPlateau': lr_scheduler = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, verbose=1, mode='auto', min_delta=0.0001, min_lr=0.001) elif self.lr_scheduler['name'] == 'LearningRateScheduler': rate = self.lr_scheduler['rate'] # reducing the learning rate by "rate" every 2 epochs lr_scheduler = LearningRateScheduler( lambda epoch: self.init_lr * rate**(epoch // 2), verbose=0) callbacks = [early_stopping, lr_scheduler] + [csv_logger] return callbacks
def get_lr_scheduler(lr: float, total_epochs: int, lr_params: dict): lr_schedule_name = lr_params['strategy'].lower() get_lr = { "exponential_decay": LearningRateScheduler(ExponentialDecay(lr)), "step_decay": LearningRateScheduler(StepDecay( lr, lr_params['decay_rate'], lr_params['drop_after_num_epoch'])), "step_decay_schedule": LearningRateScheduler(StepDecaySchedule( lr, lr_params['drop_schedule'], lr_params['decay_rate'], total_epochs)), "explicit_schedule": LearningRateScheduler(ExplicitSchedule( lr, lr_params['drop_schedule'], lr_params['list_lr'], )), "polynomial_decay": LearningRateScheduler(PolynomialDecay( lr, lr_params['power'], total_epochs)), "inverse_time_decay": LearningRateScheduler(InverseTimeDecay( lr, lr_params['decay_rate'], lr_params['decay_step'], lr_params['staircase'])), "cosine_decay": LearningRateScheduler(CosineDecay( lr, lr_params['alpha'], total_epochs)), "lr_reduce_on_plateau": ReduceLROnPlateau( monitor='val_loss', factor=lr_params['decay_rate'], patience=lr_params['patience'], verbose=1, mode='auto', min_lr=lr_params['min_lr']) } return get_lr[lr_schedule_name]
def learning_rate_scheduler(max_learn_rate, end_learn_rate, warmup_epoch_count, total_epoch_count): """ Wrapper function to return keras learning rate scheduler callback Args: max_learn_rate (float): maximum possible learning rate (achieved at peak of warmup epochs) end_learn_rate (float): minimum learning rate (achieved at end of maximum training epochs) warmup_epoch_count (int): number of epochs where learning rate rises total_epoch_count (int): maximum training epochs Returns: (tensorflow.keras.callbacks.LearningRateScheduler): LR-scheduler with internally passed learning rates """ def lr_scheduler(epoch): """ Output current learning rate based on epoch count, warmup and exponential decay Args: epoch (int): current epoch number Returns: (float): current learning rate """ if epoch < warmup_epoch_count: res = (max_learn_rate / warmup_epoch_count) * (epoch + 1) else: res = max_learn_rate * math.exp( math.log(end_learn_rate / max_learn_rate) * (epoch - warmup_epoch_count + 1) / (total_epoch_count - warmup_epoch_count + 1)) return float(res) return LearningRateScheduler(lr_scheduler, verbose=1)
def train(self): lr = self.config.learning_rate self.histories = {} earlystopping = EarlyStopping(monitor='val_loss', patience=10,min_delta=0.01, mode='auto') csv_log = CSVLogger("logs/logfiles/results.csv") # checkpoint = ModelCheckpoint( filepath='./ckpt/', save_freq='epoch',save_best_only=False, save_weights_only=True,verbose=1) self.model.compile( optimizer=tf.keras.optimizers.Adam(lr=0.0001), loss=tf.keras.metrics.binary_crossentropy, metrics=[self.macro_f1] ) for k in range(0, 1): best_val_loss = 0.5 print("Processing Fold {}.... ".format(k)) filepath='./ckpt/'+self.timestamp + '/'+str(k)+'/' # checkpoint = ModelCheckpoint( # filepath=filepath, monitor='val_loss', # save_best_only=True, save_weights_only=True,verbose=10 # ) with tf.device('/gpu:0'): history = self.model.fit( x=self.dataset[k]['train'], epochs=self.config.epochs, validation_data=self.dataset[k]['valid'], callbacks=[self.tensorboard_callback, earlystopping, csv_log, LearningRateScheduler(self.lr_decay, verbose=1)] ) print("Processing Done for {} Fold .... ".format(k)) self.histories[k] = history val_loss = history.history['val_loss'] if float(min(val_loss)) < float(best_val_loss): self.model.save(filepath) return self.model
def fit_data(self, show_figures=True): checkpointer = ModelCheckpoint(filepath=self.best_weights(self.name), verbose=self.verbose, save_best_only=True) train_values = {c: self.X_train[c] for c in self.categorical_columns} train_values['continuouse'] = self.X_train[ self.non_categorical_columns] train_values['rnn_input'] = self.ts_train val_values = {c: self.X_valid[c] for c in self.categorical_columns} val_values['continuouse'] = self.X_valid[self.non_categorical_columns] val_values['rnn_input'] = self.ts_valid lrate = LearningRateScheduler(self.step_decay, verbose=2) history = self.model.fit( train_values, self.y_train, epochs=self.epochs, batch_size=self.batch_size, validation_data=(val_values, self.y_valid), verbose=self.verbose, shuffle=True, #validation_split=0.2, callbacks=[checkpointer, lrate]) if show_figures: fig, ax = plt.subplots(figsize=(10, 5)) # plot history ax.plot(history.history['loss'], label='train') ax.plot(history.history['val_loss'], label='test') ax.legend() figure_name = os.path.join(self.output_directory, self.name + "_history.png") plt.savefig(figure_name) plt.show() self.model.load_weights(self.best_weights(self.name)) p = self.model.evaluate(val_values, self.y_valid) self.info(f"Validation Score: {p}")
def main(): inputShape = (None, None, 3) batchSize = 2 hr_image = load_img(os.path.join(os.getcwd(), 'hr_image', 'HR.png')) #target_size = inputShape[:-1]) ## loading the high-resolution image hr_image = np.array(hr_image, dtype=np.float32) * (2 / 255) - 1 hr_image = np.array([hr_image] * batchSize) ## creating fake batches lr_image = load_img(os.path.join(os.getcwd(), 'lr_image', 'LR.png')) #target_size = inputShape[:-1]) ## loading the low-resolution image lr_image = np.array(lr_image, dtype=np.float32) * (2 / 255) - 1 lr_image = np.array([lr_image] * batchSize) nn = model(inputShape) print(nn.summary()) #lr_schedule = ExponentialDecay( # initial_learning_rate=1e-1, # decay_steps=1000, # decay_rate=0.99) lrate = LearningRateScheduler(step_decay) #optimizer = SGD(learning_rate = 0.01) #optimizer = SGD(learning_rate=0.001,epsilon = 1e-9, beta_1 = .9, beta_2 = .999) optimizer = Adam(lr=0.001, epsilon=1e-8, beta_1=.9, beta_2=.999) nn.compile(optimizer=optimizer, loss='mse') #es = EarlyStopping(monitor = 'loss' , mode = 'min', verbose = 1, # patience = 100) ## early stopping to prevent overfitting history = nn.fit(lr_image, hr_image, epochs=1000, batch_size=batchSize) #callbacks=[lrate]) """ reconstrucing high-resolution image from the low-resolution image """ pred = nn.predict(lr_image) pred = np.uint8((pred + 1) * 255 / 2) pred = Image.fromarray(pred[0]) pred.save("re.png")
def build_model(self): x_input = Input(shape=self.input_shape, name='input_1') backbone = tf.keras.applications.DenseNet121(include_top=False, weights='imagenet', input_tensor=x_input, input_shape=None, pooling='avg', classes=1) # backbone = UNIWARD(x_input).model for layer in backbone.layers: layer.trainable = True out = Dense(self.num_classes, kernel_initializer='normal')(backbone.output) if self.mode == "multiclass": out = Activation('softmax')(out) model = Model(inputs=backbone.input, outputs=out) model_checkpoint = ModelCheckpoint( filepath=self.log_dir + '/regression_epoch-{epoch:02d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) tb = TensorBoard(log_dir=self.log_dir, update_freq='epoch', profile_batch=100000000) self.callbacks = [tb, model_checkpoint] adam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule, verbose=1) model.compile(optimizer=adam, loss=self.loss, metrics=[self.loss, alaska_tf]) self.model = model return self.model
def basic_callbacks(checkpoint="keras_checkpoints.h5", evals=[], lr=0.001, lr_decay=0.05, lr_min=0, lr_decay_steps=0): checkpoint_base = "./checkpoints" if not os.path.exists(checkpoint_base): os.mkdir(checkpoint_base) checkpoint = os.path.join(checkpoint_base, checkpoint) model_checkpoint = ModelCheckpoint(checkpoint, verbose=1) # model_checkpoint = keras.callbacks.experimental.BackupAndRestore(checkpoint_base) if isinstance(lr_decay_steps, list): # Constant decay on epoch lr_scheduler = ConstantDecayScheduler(sch=lr_decay_steps, lr_base=lr, decay_rate=lr_decay) elif lr_decay < 1: # Exponential decay warmup = 10 if lr_decay_steps == 0 else lr_decay_steps lr_scheduler = LearningRateScheduler(lambda epoch: scheduler(epoch, lr, lr_decay, lr_min, warmup=warmup)) else: # Cosine decay on epoch / batch lr_scheduler = CosineLrScheduler( lr_base=lr, decay_steps=lr_decay, lr_min=lr_min, warmup_iters=1, lr_on_batch=lr_decay_steps, restarts=4 ) my_history = My_history(os.path.splitext(checkpoint)[0] + "_hist.json", evals=evals) # tensor_board_log = keras.callbacks.TensorBoard(log_dir=os.path.splitext(checkpoint)[0] + '_logs') return [model_checkpoint, lr_scheduler, my_history, Gently_stop_callback()]
def get_callbacks(self, model_prefix='Model'): """ Creates a list of callbacks that can be used during training to create a snapshot ensemble of the model. Args: model_prefix: prefix for the filename of the weights. Returns: list of 3 callbacks [ModelCheckpoint, LearningRateScheduler, SnapshotModelCheckpoint] which can be provided to the 'fit' function """ if not os.path.exists('weights/'): os.makedirs('weights/') callback_list = [ModelCheckpoint('weights/%s-Best.h5' % model_prefix, monitor='val_acc', save_best_only=True, save_weights_only=True), LearningRateScheduler(schedule=self._cosine_anneal_schedule), SnapshotModelCheckpoint(self.T, self.M, fn_prefix='weights/%s' % model_prefix)] return callback_list
def create_callbacks(log_dir): # early_stopping = EarlyStopping(patience=4, monitor='val_loss', verbose=1) lr_schedule = LearningRateScheduler(clr.lrfn, verbose=1) model_name = f'./output/models/best-model.hdf5' model_checkpoint = ModelCheckpoint(monitor='val_loss', filepath=model_name, save_best_only=True, verbose=1, pooling='average') tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) callbacks = [ lr_schedule, model_checkpoint, tensorboard_callback, ] return callbacks
def train(X_train): all_predictions = [] from sklearn.model_selection import KFold import tensorflow.keras.callbacks.LearningRateScheduler import math kf = KFold(n_splits=5, random_state=42, shuffle=True) lr_schedule = LearningRateScheduler(lambda epoch: 1e-8 * math.pow(drop, math.floor((1+epoch)/0.001))) for ind, (tr, val) in enumerate(kf.split(X_train)): X_tr = X_train[tr] y_tr = y_train[tr] X_vl = X_train[val] y_vl = y_train[val] model = create_model() print( X_tr.shape,y_tr.shape,X_vl.shape,y_vl.shape) model.fit( X_tr, y_tr, epochs=1, batch_size=64, validation_data=(X_vl, y_vl), verbose=True, callbacks=[lr_schedule] ) print("Done training! Now predicting") all_predictions.append(model.predict(X_test)) return all_predictions
def cifar10_classfication_train_procedure_with_alexnet(save_path, epoch=5): import Data import Model def lr_schedule(epoch_index, cur_lr): if epoch_index < 100: return 0.001 elif epoch_index < 120: return 0.02 else: return 0.02 def image_preprocessing(images): return images start_time = time.time() x_train, y_train, x_test, y_test = Data.load_data( dataset_name="cifar10", data_dir="./tmp/cifar-10-batches-py/", preprocessing=image_preprocessing) end_time = time.time() print("[Train] Load data time: ", end_time - start_time) model = Model.alexnet(dataset_name="cifar10", shape=(32, 32, 3)) model.summary() lr_scheduler = LearningRateScheduler(lr_schedule, verbose=1) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=epoch, batch_size=32, callbacks=[lr_scheduler]) model.save(save_path)
def train(self, X_train, y_train, X_val, y_val): # コンパイル self.model.compile(optimizer=SGD(lr=self.initial_lr, momentum=0.9), loss="categorical_crossentropy", metrics=["acc"]) # Data Augmentation traingen = ImageDataGenerator(rescale=1. / 255, width_shift_range=4. / 32, height_shift_range=4. / 32, horizontal_flip=True) valgen = ImageDataGenerator(rescale=1. / 255) # TPU tpu_grpc_url = "grpc://" + os.environ["COLAB_TPU_ADDR"] tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( tpu_grpc_url) strategy = keras_support.TPUDistributionStrategy(tpu_cluster_resolver) self.model = tf.contrib.tpu.keras_to_tpu_model(self.model, strategy=strategy) # Callback time_cb = TimeHistory() lr_cb = LearningRateScheduler(self.lr_schduler) # Train history = self.model.fit_generator(traingen.flow(X_train, y_train, batch_size=1024), epochs=self.nb_epochs, steps_per_epoch=len(X_train) / 1024, validation_data=valgen.flow( X_val, y_val, batch_size=1024), validation_steps=len(X_val) / 1024, callbacks=[time_cb, lr_cb]).history history["time"] = time_cb.times # Save history file_name = f"{self.framework}_n{self.n}.dat" with open(file_name, "wb") as fp: pickle.dump(history, fp)
def set_callbacks(self): """ registers the callbacks * EarlyStopping: stops the model when it no longer learns * ModelChckpoint: store the wights of the best epoch * ReduceLROnPlateau: reduce LR when it find itself in a plateu * LearningRateScheduler: halves the LR every 5th epoch :return: """ print(self.weights_path) checkpoint = ModelCheckpoint(self.weights_path, monitor='dice_coef', verbose=1, save_best_only=True, mode='max', save_weights_only=True) reduceLROnPlat = ReduceLROnPlateau(monitor='dice_coef', factor=0.33, patience=3, verbose=1, mode='max', epsilon=0.0001, cooldown=0, min_lr=1e-8) early = EarlyStopping(monitor="dice_coef", mode="max", patience=50) def scheduler(epoch, lr): e = epoch + 1 if e % 5 != 0: new_lr = lr else: new_lr = lr * 0.5 print("Reducing learning rate to: " + str(new_lr)) return new_lr lr_schedule = LearningRateScheduler(schedule=scheduler) self.callbacks_list = [checkpoint, reduceLROnPlat, early, lr_schedule]
def train_network(): history = load_data() xs, y_policies, y_values = zip(*history) # 学習のための入力データのシェイプの変換 a, b, c = DN_INPUT_SHAPE xs = np.array(xs) xs = xs.reshape(len(xs), c, a, b).transpose(0, 2, 3, 1) y_policies = np.array(y_policies) y_values = np.array(y_values) model = load_model('./model/best.h5') model.compile(loss=['categorical_crossentropy', 'mse'], optimizer='adam') # 学習率 def step_decay(epoch): x = 0.001 if epoch >= 50: x = 0.0005 if epoch >= 80: x = 0.00025 return x lr_decay = LearningRateScheduler(step_decay) print_callback = LambdaCallback(on_epoch_begin=lambda epoch, logs: print( '\rTrain {}/{}'.format(epoch + 1, RN_EPOCHS), end='')) model.fit(xs, [y_policies, y_values], batch_size=128, epochs=RN_EPOCHS, verbose=0, callbacks=[lr_decay, print_callback]) print('') model.save('./model/latest.h5') K.clear_session() del model
def train_model(): generator = data_generator.SegmentationSequence( os.path.normpath("..\\data\\train\\images"), os.path.normpath("..\\data\\train\\masks"), augmenter=augmentation.augment, batch_size=4, output_img_size=(240, 240), steps_per_epoch=1000, ) model = model_builder.build_segmentation_model((240, 240, 3), 4, 4, 20, 16) opt = tf.keras.optimizers.Adam(lr=lr_schedule(0), clipnorm=1.0) model.compile( optimizer=opt, loss=tversky_loss, metrics=[dice_coef(1.0)], ) tb = TensorBoard( log_dir=os.path.normpath(".\\logs\\fit\\") + datetime.datetime.now().strftime("%Y%m%d-%H%M%S"), histogram_freq=1, ) # lr_reducer = ReduceLROnPlateau( # factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-8 # ) lr_scheduler = LearningRateScheduler(lr_schedule) checkpoints = ModelCheckpoint( os.path.normpath(".\\ckpts"), monitor="loss", verbose=0, save_weights_only=False, mode="auto", save_freq="epoch", ) callbacks = [lr_scheduler, tb, checkpoints] history = model.fit(generator, epochs=EPOCHS, callbacks=callbacks) model.save(MODEL_SAVE_PATH)
def run_lr_sweep(model, x, y=None, lr_start=1e-6, lr_stop=1e2, steps=1000, mode='exponential', cutoff=None): """ Takes a compiler keras model and runs a learning rate sweep with some data. Will plot Loss and Learning Rate after completed run. :param model: Compiled Keras model :param x: either data generator or training input :param y: None if x is a data generator, else training targets :param lr_start: Lower bound for learning rate :param lr_stop: Higher bound for learning rate :param steps: Number of steps to perform learning sweep over. In effect this is the number of epochs with steps_per_epoch=1 :param mode: Rate of change for learning rate. Either 'linear' or 'exponential'. :param cutoff: When loss explodes, the plotting will try to clip that part of the plot so smaller differences in loss is visible without extensive zooming. Optionally, set this argument to limit the loss-axis in all plots """ data = GatherLRDataCallback() sweep_func = _get_lr_sweep(lr_start, lr_stop, steps, mode) # plt.plot([sweep_func(i) for i in range(steps)]) # # plt.xlabel('steps') # plt.ylabel('LR') # plt.show() model.fit(x if y is None else (x, y), epochs=steps, steps_per_epoch=1, shuffle=False, callbacks=[data, LearningRateScheduler(sweep_func)]) data.plot(cutoff=cutoff)
def capsul_train_predict(): (x_train, y_train), (x_test, y_test), (x_indt, y_indt) = load_data(col=9, row=11) model = CapsNet(input_shape=(11,10,2,),kerl=3, n_class=2, num_routing=5) model.compile(optimizer=Adam(learning_rate=lr_schedule(0)), loss=[margin_loss, 'mse'], loss_weights=[1., 0.325], metrics={'out_caps': 'accuracy'}) lr_scheduler = LearningRateScheduler(lr_schedule) lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6) save_dir = os.path.join(os.getcwd(), 'save_models') model_name = 'trb_capsul_model.h5' if not os.path.isdir(save_dir): os.makedirs(save_dir) filepath = os.path.join(save_dir, model_name) checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_out_caps_accuracy', verbose=1, save_best_only=True, save_weights_only=True) cbs = [checkpoint, lr_reducer, lr_scheduler] model.fit([x_train, y_train], [y_train, x_train], batch_size=32, epochs=10, validation_data=[[x_test, y_test], [y_test, x_test]], shuffle=True, callbacks=cbs) model.load_weights(filepath) y_pred, _ = model.predict([x_test, y_test]) info = 'By capsul net Training in TRB, Testing in testdata.....' writeMetrics('TRB_result.txt', y_test, y_pred, info) indt_pred, _ = model.predict([x_indt, y_indt]) info = 'By capsul net Tringing in TRB, Testing in nCoV-19...' writeMetrics('TRB_result.txt', y_indt, indt_pred, info)
def train(model: tf.keras.models) -> tf.keras.models: logdir = tempfile.mkdtemp() print('Writing training logs to ' + logdir) keras_file = '{0}/{0}_{1}_temp.h5'.format(args.architecture, args.dataset) tb = tf.keras.callbacks.TensorBoard(log_dir=logdir, profile_batch=0) es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=round(cfg.epochs / 4)) mc = ModelCheckpoint(keras_file, monitor='val_acc', mode='max', verbose=1, save_best_only=True) learning_rate_scheduler = LearningRateScheduler( schedule=cfg.learning_rate_scheduler) callbacks = [tb, es, mc, learning_rate_scheduler] opt = tf.keras.optimizers.Adam() model.compile(loss=tf.keras.losses.categorical_crossentropy, optimizer=opt, metrics=['accuracy']) model.fit(x_train, y_train, batch_size=cfg.batch_size, epochs=cfg.epochs, verbose=1, callbacks=callbacks, validation_data=(x_test, y_test)) score = model.evaluate(x_test, y_test, verbose=1) print('Test loss:', score[0]) print('Test accuracy:', score[1]) print('Saving model to: ', keras_file.replace('temp', 'original')) tf.keras.models.save_model(model, keras_file.replace('temp', 'original'), include_optimizer=False) return model