def update_learning_rate(self, step, lr, optimizers): """Since we cannot use fit, we have to update the learning rate manually. Args: step (int): current step. lr: current learning rate optimizers: list of optimizers to change """ cur_lr = self._calculate_lr(step, lr) for _optimizer in optimizers: K.set_value(_optimizer.learning_rate, cur_lr)
def on_epoch_begin(self, epoch, logs=None): if not hasattr(self.model.optimizer, 'lr'): raise ValueError('Optimizer must have a "lr" attribute.') step = min(epoch, self.n_epoch) cosine_decay = 0.5 * (1 + cos(pi * step / self.n_epoch)) decayed = (1 - self.alpha) * cosine_decay + self.alpha lr = self.initial_learning_rate * decayed K.set_value(self.model.optimizer.lr, lr) if self.verbose > 0: print('\nEpoch %05d: CosineAnnealingScheduler setting learning ' 'rate to %s.' % (epoch + 1, lr))
def on_batch_begin(self, batch, logs=None): self.steps += 1 lr = L_FACTOR * min(1.0, self.steps / self.warm) / max( self.steps, self.warm) K.set_value(self.model.optimizer.lr, lr) if os.path.isfile('stop'): print("Stop file found.") global stop stop = True self.model.stop_training = True mdl.save_weights("models/final.h5", save_format="h5")
def setAllParams(self, list_of_values): """ Set all parameters used by the learning algorithm Arguments --------- list_of_values : list of numpy arrays list of the parameters to be set (same order than given by getAllParams()). """ for i, p in enumerate(self.params): K.set_value(p, list_of_values[i]) for j, p in enumerate(self.params_policy): K.set_value(p, list_of_values[j + i + 1])
def on_epoch_end(self, epoch, logs=None): if not hasattr(self.model, "encoder"): raise ValueError('Model must have a "encoder" attribute.') if not hasattr(self.model.encoder, "kld_weight"): raise ValueError( 'Model encoder must have a "kld_weight" attribute.') kld_weight = float(K.get_value(self.model.encoder.kld_weight)) assert kld_weight <= self.max_val delta = self.max_val - kld_weight kld_weight = min(kld_weight + delta * epoch, self.max_val) K.set_value(self.model.encoder.kld_weight, K.get_value(kld_weight))
def auto_encoder_transform(Inputs, Outputs, n_dims=32, verbose=0, epochs=10): import tensorflow as tf import tensorflow.keras.backend as K from tensorflow import keras loss = "mean_squared_error" es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0, patience=5, verbose=0, ) reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=4, min_lr=0.0001) nan_prevent = tf.keras.callbacks.TerminateOnNaN() input_layer = tf.keras.Input(shape=(Inputs.shape[1],)) encoded = tf.keras.layers.Dense(n_dims * 4, activation='relu', kernel_regularizer=keras.regularizers.l1_l2(l2=1e-5), use_bias=False)(input_layer) encoded = tf.keras.layers.Dense(n_dims, activation='linear', use_bias=False)(encoded) encoded = K.l2_normalize(encoded, axis=-1) decoded = tf.keras.layers.Dense(n_dims * 4, activation='relu', kernel_regularizer=keras.regularizers.l1_l2(l2=1e-5), use_bias=False)(encoded) decoded = tf.keras.layers.Dense(Outputs.shape[1], use_bias=False)(decoded) autoencoder = tf.keras.Model(input_layer, decoded) encoder = tf.keras.Model(input_layer, encoded) adam = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) autoencoder.compile(optimizer=adam, loss=loss, metrics=["mean_squared_error"]) Inputs = Inputs.astype(float) Outputs = Outputs.astype(float) X1, X2, Y1, Y2 = train_test_split(Inputs, Outputs, test_size=0.5) autoencoder.fit(X1, Y1, epochs=epochs, batch_size=2048, shuffle=True, verbose=verbose, validation_data=(X2, Y2), callbacks=[es, reduce_lr, nan_prevent]) K.set_value(autoencoder.optimizer.lr, 0.001) es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0, patience=5, verbose=0, ) reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.2, patience=4, min_lr=0.00001) autoencoder.fit(X2, Y2, epochs=epochs, batch_size=2048, shuffle=True, verbose=verbose, validation_data=(X1, Y1), callbacks=[es, reduce_lr, nan_prevent]) K.set_value(autoencoder.optimizer.lr, 0.001) reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor="loss", factor=0.2, patience=1, min_lr=0.00001) autoencoder.fit(Inputs, Outputs, epochs=epochs, batch_size=2048, shuffle=True, verbose=verbose, callbacks=[reduce_lr, nan_prevent]) Z = encoder.predict(Inputs) return Z, encoder
def on_batch_begin(self, batch, logs=None): lr = cosine_decay_with_warmup( global_step=self.global_step, learning_rate_base=self.learning_rate_base, total_steps=self.total_steps, warmup_learning_rate=self.warmup_learning_rate, warmup_steps=self.warmup_steps, hold_base_rate_steps=self.hold_base_rate_steps) K.set_value(self.model.optimizer.lr, lr) if self.verbose > 0: print('\nBatch %05d: setting learning ' 'rate to %s.' % (self.global_step + 1, lr))
def on_epoch_end(self, epoch, logs={}): if (epoch < self.epochs_max_point): self.lr = self.lr + self.lr_step_size elif (epoch >= self.epochs_max_point and epoch < self.epochs_max_point * 2): self.lr = self.lr - self.lr_step_size else: self.lr = self.min_lr K.set_value(self.model.optimizer.lr, self.lr) self.lrs.append(self.lr)
def on_batch_end(self, batch, logs=None): self.counter += 1 if self.counter % 1000 == 500: if not hasattr(self.model.optimizer, 'lr'): raise ValueError('Optimizer must have a "lr" attribute.') self.x = min([math.pi, (self.counter / self.half_period) * math.pi]) self.lr = self.min_lr + (self.initial_lr - self.min_lr) * (1.0 + math.cos(self.x)) / 2.0 K.set_value(self.model.optimizer.lr, self.lr)
def train(self, x_label, y_label, learning_rate=None): board_input = np.array(x_label) probs, values = y_label probs_output = np.array(probs) values_output = np.array(values) K.set_value(self.model.optimizer.lr, learning_rate) self.model.fit(board_input, [probs_output, values_output], batch_size=len(x_label), verbose=0)
def set_ig_values(self, delta_value, edge_value): """ Set values of the integrated gradient parameters in all layers of the model. Args: delta_value: Value of the `delta` parameter edge_value: Value of the `non_exist_edges` parameter """ for delta_var in self.deltas: K.set_value(delta_var, delta_value) for edge_var in self.non_exist_edges: K.set_value(edge_var, edge_value)
def optimizer_params_step(self): next_lr = self.lr_scheds[self.sched_idx].step() next_momentum = self.momentum_scheds[self.sched_idx].step() # add to logs self.logs.setdefault('lr', []).append(next_lr) self.logs.setdefault('momentum', []).append(next_momentum) # update optimizer params K.set_value(self.model.optimizer.lr, next_lr) if hasattr(self.model.optimizer, 'momentum'): K.set_value(self.model.optimizer.momentum, next_momentum)
def on_train_begin(self, logs={}): logs = logs or {} self._reset() K.set_value(self.model.optimizer.lr, self.compute_lr()) if self._update_momentum: if not hasattr(self.model.optimizer, 'momentum'): raise ValueError("Momentum can be updated only on SGD optimizer !") new_momentum = self.compute_momentum() K.set_value(self.model.optimizer.momentum, new_momentum)
def on_epoch_begin(self, epoch, logs=None): if not hasattr(self.model.optimizer, 'lr'): raise ValueError('Optimizer must have a "lr" attribute.') learning_rate = float(K.get_value(self.model.optimizer.lr)) learning_rate = self.schedule(epoch) if not isinstance(learning_rate, (float, np.float32, np.float64)): raise ValueError('Learning rate should be float.') K.set_value(self.model.optimizer.lr, learning_rate) if self.verbose > 0: print('\nEpoch %05d: LearningRateScheduler reducing learning ' 'rate to %s.' % (epoch + 1, learning_rate))
def update_lr(self): if not hasattr(self.model.optimizer, "lr"): raise ValueError('Optimizer must have a "lr" attribute.') lr = float(K.get_value(self.model.optimizer.lr)) lr = lr * self.lr_multiplier if not isinstance(lr, (float, np.float32, np.float64)): raise ValueError('The output of the "schedule" function ' "should be float.") K.set_value(self.model.optimizer.lr, lr) if self.verbose > 0: print("\nBatch %05d: LR_Finder reducing learning " "rate to %s." % (self.batch_num + 1, lr))
def mc_dropout(self, batch_size=1000, dropout=0.5, iter=100): K.set_value(self.mc_dropout_rate, dropout) repititions = [] for i in range(iter): _, pred = self.model.predict(self.x_test, batch_size) repititions.append(pred) K.set_value(self.mc_dropout_rate, 0) repititions = np.array(repititions) mc = np.var(repititions, 0) mc = np.mean(mc, -1) return -mc
def on_batch_end(self, epoch, logs=None): '''Record previous batch statistics and update the learning rate.''' logs = logs or {} self.iteration += 1 self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr)) self.history.setdefault('iterations', []).append(self.iteration) for k, v in logs.items(): self.history.setdefault(k, []).append(v) K.set_value(self.model.optimizer.lr, self.clr())
def on_epoch_begin(self, epoch, logs=None): if logs is None: logs = {} new_weight = self.schedule(epoch) new_value = new_weight * self.weight_orig print("Current {} annealer weight is {}".format( self.weight_name, new_value)) assert type( new_weight ) == float, 'The output of the "schedule" function should be float.' K.set_value(self.weight_var, new_value)
def on_batch_begin(self, batch, logs=None): new_kl_weight = self.schedule(self.count) if not isinstance(new_kl_weight, (float, np.float32, np.float64)): raise ValueError( 'The output of the "schedule" function should be float.') # Set new value K.set_value(self.kl_weight, new_kl_weight) if self.verbose > 0 and self.count % 20 == 0: print('\nBatch %05d: KLWeightScheduler setting KL weight to %s.' % (self.count + 1, new_kl_weight)) self.count += 1
def on_batch_begin(self, batch, logs=None): 'TODO: docstring' if not hasattr(self.model.optimizer, 'lr'): raise ValueError('Optimizer must have a "lr" attribute.') if not hasattr(self.model.optimizer, 'momentum'): raise ValueError('Optimizer must have a "momentum" attribute.') lr = _1cycle_lr(self.iteration, self.cyc_iterations, self.ramp_iterations, self.min_lr, self.max_lr) mom = _1cycle_mom(self.iteration, self.cyc_iterations, self.min_mom, self.max_mom) K.set_value(self.model.optimizer.lr, lr) K.set_value(self.model.optimizer.momentum, mom)
def on_batch_end(self, model, logs=None): logs = logs or {} self.trn_iterations += 1 self.clr_iterations += 1 self.history.setdefault('lr', []).append(K.get_value(model.optimizer.lr)) self.history.setdefault('iterations', []).append(self.trn_iterations) for k, v in logs.items(): self.history.setdefault(k, []).append(v) K.set_value(model.optimizer.lr, self.clr())
def on_epoch_end(self, epoch, logs=None): logs = logs or {} self.trn_iterations += 1 self.clr_iterations += 1 print("epoch : ", epoch) self.history.setdefault('lr', []).append( K.get_value(self.model.optimizer.lr)) self.history.setdefault('iterations', []).append(self.trn_iterations) for k, v in logs.items(): self.history.setdefault(k, []).append(v) if self.clr_iterations > self.lr_cg_th: K.set_value(self.model.optimizer.lr, self.htdlr())
def on_epoch_begin(self, epoch, logs): if not hasattr(self.model.optimizer, 'lr'): raise ValueError('Optimizer must have a "lr" attribute\n') lr = self.eta_min + (self.eta_max - self.eta_min) * ( 1 + math.cos(math.pi * epoch / self.T_max)) / 2 K.set_value(self.model.optimizer.lr, lr) if self.verbose > 0: print( f'\nEpoch {(epoch+1):5d} CosineAnnealingScheduler - setting learning rate to {lr}' )
def on_batch_begin(self, batch, logs=None): """第一个epoch用来warmup,第二个epoch把学习率降到最低 """ if self.passed < self.params['steps']: lr = (self.passed + 1.) / self.params['steps'] * self.learning_rate K.set_value(self.model.optimizer.lr, lr) self.passed += 1 elif self.params['steps'] <= self.passed < self.params['steps'] * 2: lr = (2 - (self.passed + 1.) / self.params['steps']) * ( self.learning_rate - self.min_learning_rate) lr += self.min_learning_rate K.set_value(self.model.optimizer.lr, lr) self.passed += 1
def reset_optimizers(res_log2): """Reset the optimizer state for the next resolution of training. Also adjust LR.""" with strategy.scope(): lr = learning_rate * (learning_rate_decay**(res_log2 - 2)) for opt in [G_optimizer, D_optimizer]: weights = opt.get_weights() opt.set_weights([ 0.0 if isinstance(w, float) else np.zeros(w.shape, dtype=w.dtype) for w in weights ]) K.set_value(opt.iterations, 0) K.set_value(opt.lr, lr)
def on_batch_begin(self, batch, logs): # keep track of total batch count self._cur_iter += 1 # calculate current learning rate lr = self._lr_init * (1 - (self._cur_iter / self._max_iter))**self._power # limit learning rate lr = max(lr, self._lr_min) # set the new learning rate K.set_value(self.model.optimizer.lr, lr)
def on_batch_begin(self, batch, logs=None): # params是模型自动传递给Callback的一些参数 if self.params['steps'] == None: self.steps_per_epoch = np.ceil(1. * self.params['samples'] / self.params['batch_size']) else: self.steps_per_epoch = self.params['steps'] if self.num_passed_batchs < self.steps_per_epoch * self.warmup_epochs: # 前10个epoch中,学习率线性地从零增加到0.001 K.set_value( self.model.optimizer.lr, 0.001 * (self.num_passed_batchs + 1) / self.steps_per_epoch / self.warmup_epochs) self.num_passed_batchs += 1
def update_weights(self): new_weights = list() for name in self.loss_weights.keys(): K.set_value( self.loss_weights[name], self.rolling_avg[self.ref_loss] / self.rolling_avg[name]) new_weights.append(self.rolling_avg[self.ref_loss] / self.rolling_avg[name]) out_str = '' for name, val in zip(self.loss_weights.keys(), new_weights): out_str += '{}: {:7.2f}\t'.format(name, val) print('WEIGHTS UPDATE: ' + out_str)
def on_batch_begin(self, batch, logs=None): if self.should_update: if self.current_step < self.warmup_steps: lr = self.warmup(current_step=self.current_step) else: lr = self.final_lr self.should_update = False K.set_value(self.model.optimizer.lr, lr) self.current_step += 1 if self.verbose > 0: print(f'\nSetting learning rate to {lr}')
def on_batch_end(self, batch, logs=None): logs = logs or {} if self.step < self.w: lr = self.lr / self.w * self.step else: lr = K.get_value(self.optimizer.lr) for step in self.p: if step == self.step: lr = lr / self.factor K.set_value(self.optimizer.lr, lr) self.step += 1