def get_config(self): config = {'lr': float(K.get_value(self.lr)), 'momentum': float(K.get_value(self.momentum)), 'decay': float(K.get_value(self.decay)), 'nesterov': self.nesterov} base_config = super(SimpleSGD, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def get_config(self): config = { 'scale_factor': float(K.get_value(self.scale_factor)), 'shift_factor': float(K.get_value(self.shift_factor)), } base_config = super(AddLighting, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def save_weights(self, filepath, overwrite=False): # Save weights to HDF5 import h5py import os.path # if file exists and should not be overwritten if not overwrite and os.path.isfile(filepath): import sys get_input = input if sys.version_info[:2] <= (2, 7): get_input = raw_input overwrite = get_input('[WARNING] %s already exists - overwrite? [y/n]' % (filepath)) while overwrite not in ['y', 'n']: overwrite = get_input('Enter "y" (overwrite) or "n" (cancel).') if overwrite == 'n': return print('[TIP] Next time specify overwrite=True in save_weights!') f = h5py.File(filepath, 'w') biases = [K.get_value(x) for x in self.bs] weights= [K.get_value(w) for w in self.Ws] f.attrs['nb_params'] = len(weights) for n, param in enumerate(biases): param_name = 'param_bias_{}'.format(n) param_dset = f.create_dataset(param_name, param.shape, dtype=param.dtype) param_dset[:] = param for n, param in enumerate(weights): param_name = 'param_weight_{}'.format(n) param_dset = f.create_dataset(param_name, param.shape, dtype=param.dtype) param_dset[:] = param f.flush() f.close()
def on_epoch_end(self, epoch, logs={}): logs['lr'] = K.get_value(self.model.optimizer.lr) current = logs.get(self.monitor) if current is None: warnings.warn('Learning Rate Plateau Reducing requires %s available!' % self.monitor, RuntimeWarning) else: if self.in_cooldown(): self.cooldown_counter -= 1 self.wait = 0 if self.monitor_op(current, self.best): self.best = current self.wait = 0 elif not self.in_cooldown(): if self.wait >= self.patience: old_lr = float(K.get_value(self.model.optimizer.lr)) if old_lr > self.min_lr + self.lr_epsilon: new_lr = old_lr * self.factor new_lr = max(new_lr, self.min_lr) K.set_value(self.model.optimizer.lr, new_lr) if self.verbose > 0: print('\nEpoch %05d: reducing learning rate to %s.' % (epoch, new_lr)) self.cooldown_counter = self.cooldown self.wait = 0 self.wait += 1
def test_ReduceLROnPlateau(): np.random.seed(1337) (X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples, num_test=test_samples, input_shape=(input_dim,), classification=True, num_classes=num_class) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) def make_model(): np.random.seed(1337) model = Sequential() model.add(Dense(num_hidden, input_dim=input_dim, activation='relu')) model.add(Dense(num_class, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=optimizers.SGD(lr=0.1), metrics=['accuracy']) return model model = make_model() # This should reduce the LR after the first epoch (due to high epsilon). cbks = [callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, epsilon=10, patience=1, cooldown=5)] model.fit(X_train, y_train, batch_size=batch_size, validation_data=(X_test, y_test), callbacks=cbks, epochs=5, verbose=2) assert np.allclose(float(K.get_value(model.optimizer.lr)), 0.01, atol=K.epsilon()) model = make_model() cbks = [callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, epsilon=0, patience=1, cooldown=5)] model.fit(X_train, y_train, batch_size=batch_size, validation_data=(X_test, y_test), callbacks=cbks, epochs=5, verbose=2) assert np.allclose(float(K.get_value(model.optimizer.lr)), 0.1, atol=K.epsilon())
def get_config(self): config = {'lr': float(K.get_value(self.lr)), 'momentum': float(K.get_value(self.momentum)), 'nesterov': self.nesterov, 'decay_block': float(self.decy_block)} base_config = super(SGD_step_decay, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def train(self, model, data): """ Fits the given model on a batch of data. """ kur_optimizer = model.compiled['train']['kur_optimizer'] if kur_optimizer.scale_rate: if kur_optimizer.scale_rate in data: import keras.backend as K # pylint: disable=import-error factor = numpy.mean(data[kur_optimizer.scale_rate]) if kur_optimizer.scale_mode == 'sqrt': factor = factor ** 0.5 keras_optimizer = kur_optimizer.optimizer K.set_value( keras_optimizer.lr, K.get_value(keras_optimizer.lr) * factor ) result = self.run_batch(model, data, 'train', True) K.set_value( keras_optimizer.lr, K.get_value(keras_optimizer.lr) / factor ) return result else: logger.warning('The optimizer "scale_rate" was specified, but ' 'no such data column was found: %s. Ignoring this.', kur_optimizer.scale_rate) return self.run_batch(model, data, 'train', True)
def get_config(self): config = { 'sigma': self.sigma, 'threshold': float(K.get_value(self.threshold)), 'smooth_threshold': float(K.get_value(self.smooth_threshold)), } base_config = super(Selection, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def _adjust_learning_rate(self, epoch): old_lr = K.get_value(self.model.optimizer.lr) new_lr = self.initial_lr * self.multiplier(epoch) K.set_value(self.model.optimizer.lr, new_lr) if hasattr(self.model.optimizer, 'momentum') and self.momentum_correction: # See the paper cited above for more information about momentum correction. self.restore_momentum = K.get_value(self.model.optimizer.momentum) K.set_value(self.model.optimizer.momentum, self.restore_momentum * new_lr / old_lr)
def test_clone_optimizer(): lr, momentum, clipnorm, clipvalue = np.random.random(size=4) optimizer = SGD(lr=lr, momentum=momentum, clipnorm=clipnorm, clipvalue=clipvalue) clone = clone_optimizer(optimizer) assert isinstance(clone, SGD) assert K.get_value(optimizer.lr) == K.get_value(clone.lr) assert K.get_value(optimizer.momentum) == K.get_value(clone.momentum) assert optimizer.clipnorm == clone.clipnorm assert optimizer.clipvalue == clone.clipvalue
def get_config(self): config = { "lr": float(K.get_value(self.lr)), "beta_1": float(K.get_value(self.beta_1)), "beta_2": float(K.get_value(self.beta_2)), "beta_3": float(K.get_value(self.beta_3)), "epsilon": self.epsilon, "decay": float(K.get_value(self.decay)) } base_config = super(Eve, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def on_epoch_end(self, epoch, logs=None): super(LearningRateWarmupCallback, self).on_epoch_end(epoch, logs) if epoch == self.end_epoch - 1 and self.verbose > 0: new_lr = K.get_value(self.model.optimizer.lr) print('\nEpoch %d: finished gradual learning rate warmup to %g.' % (epoch + 1, new_lr))
def on_batch_begin(self, batch, logs={}): open_all_gates() rands = np.random.uniform(size=len(add_tables)) for t, rand in zip(add_tables, rands): if rand < K.get_value(t["death_rate"]): K.set_value(t["gate"], 0)
def build(self, input_shape): input_dim = input_shape[-1] W1 = weight((input_dim, 4 * self.hidden_dim,), init=self.init, regularizer=self.W_regularizer, name='{}_W1'.format(self.name)) W2 = weight((self.hidden_dim, self.output_dim), init=self.init, regularizer=self.W_regularizer, name='{}_W2'.format(self.name)) U = weight((self.hidden_dim, 4 * self.hidden_dim,), init=self.inner_init, regularizer=self.U_regularizer, name='{}_U'.format(self.name)) b1 = np.concatenate([np.zeros(self.hidden_dim), K.get_value(self.forget_bias_init((self.hidden_dim,))), np.zeros(2 * self.hidden_dim)]) b1 = weight(b1, regularizer=self.b_regularizer, name='{}_b1'.format(self.name)) b2 = weight((self.output_dim,), init='zero', regularizer=self.b_regularizer, name='{}_b2'.format(self.name)) h = (-1, self.hidden_dim) c = (-1, self.hidden_dim) def step(x, states, weights): h_tm1, c_tm1 = states W1, W2, U, b1, b2 = weights z = K.dot(x, W1) + K.dot(h_tm1, U) + b1 z0 = z[:, :self.hidden_dim] z1 = z[:, self.hidden_dim: 2 * self.hidden_dim] z2 = z[:, 2 * self.hidden_dim: 3 * self.hidden_dim] z3 = z[:, 3 * self.hidden_dim:] i = self.inner_activation(z0) f = self.inner_activation(z1) c = f * c_tm1 + i * self.activation(z2) o = self.inner_activation(z3) h = o * self.activation(c) y = self.activation(K.dot(h, W2) + b2) return y, [h, c] self.step = step self.states = [h, c] self.weights = [W1, W2, U, b1, b2] super(LSTMCell, self).build(input_shape)
def build(self, input_shape): stack_size = input_shape[1] self.W_shape = (stack_size, self.nb_filter, self.nb_row, self.nb_col) w = self.init(self.W_shape) self.W = K.variable(K.get_value(w).reshape(self.W_shape)) self.b = K.zeros((self.nb_filter,)) self._trainable_weights = [self.W, self.b]
def get_config(self): config = { 'sigma': self.sigma, 'nb_steps': float(K.get_value(self.nb_steps)), } base_config = super(HighPass, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def ExponentialMovingAverage_TrainBegin(model): # run when training begins # ema_trainable_weights_vals save the latest weights of model with ema ema_trainable_weights_vals ={} for weight in tqdm(model.trainable_weights): ema_trainable_weights_vals[weight.name]=K.get_value(weight) return ema_trainable_weights_vals
def reduce_lr(self, current_nb): if self.reduction_function == 'linear': new_rate = self.reduce_rate elif self.reduction_function == 'exponential': new_rate = np.power(self.exp_base, current_nb / self.half_life) * self.reduce_rate elif self.reduction_function == 'noam': new_rate = np.float32(min(float(current_nb) ** self.exp_base, float( current_nb) * self.half_life ** self.warmup_exp)) else: raise NotImplementedError( 'The decay function %s is not implemented.' % str( self.reduction_function)) if self.reduction_function == 'noam': lr = self.initial_lr else: lr = K.get_value(self.model.optimizer.lr) self.new_lr = np.maximum(np.float32(lr * new_rate), self.min_lr) K.set_value(self.model.optimizer.lr, self.new_lr) if self.reduce_each_epochs and self.verbose > 0: logging.info("LR reduction from {0:0.6f} to {1:0.6f}".format(float(lr), float(self.new_lr)))
def get_numvals_list(self, key='omega'): """ Returns list of numerical values such as for instance omegas in reproducible order """ variables = self.vars[key] numvals = [] for p in self.weights: numval = K.get_value(tf.reshape(variables[p],(-1,))) numvals.append(numval) return numvals
def get_updates(self, params, constraints, loss): grads = self.get_gradients(loss, params) for param, grad, c in zip(params, grads, constraints): grad_tm1 = K.variable(np.zeros(K.get_value(param).shape)) step_tm1 = K.variable( self.init_step*np.ones(K.get_value(param).shape)) test = grad * grad_tm1 diff = T.lt(test, 0) steps = step_tm1 * (T.eq(test, 0) + T.gt(test, 0) * self.increase + diff * self.decrease) step = T.minimum(self.max_step, T.maximum(self.min_step, steps)) grad = grad - diff * grad self.updates.append((param, c(param - T.sgn(grad) * step))) self.updates.append((grad_tm1, grad)) self.updates.append((step_tm1, step)) return self.updates
def get_state(self): state = [] vs = self.vars for key in vs.keys(): if key=='oopt': continue v = vs[key] for p in v.values(): state.append(K.get_value(p)) # FIXME WhyTF does this not work? return state
def XXXget_training_function(self, x,y): # get trainable weights trainable_weights = [] for layer in self.model.layers: trainable_weights += collect_trainable_weights(layer) # get the grads - more or less weights = [K.variable(np.zeros(K.get_value(p).shape)) for p in trainable_weights] training_updates = self.optimizer.get_updates(trainable_weights, self.constraints, self.total_loss)
def test_load_model_custom_objects(self): hvd.init() class TestOptimizer(keras.optimizers.RMSprop): def __init__(self, **kwargs): super(TestOptimizer, self).__init__(**kwargs) with self.test_session() as sess: K.set_session(sess) opt = TestOptimizer(lr=0.0001) opt = hvd.DistributedOptimizer(opt) model = keras.models.Sequential() model.add(keras.layers.Dense(2, input_shape=(3,))) model.add(keras.layers.RepeatVector(3)) model.add(keras.layers.TimeDistributed(keras.layers.Dense(3))) model.compile(loss=keras.losses.MSE, optimizer=opt, metrics=[keras.metrics.categorical_accuracy], sample_weight_mode='temporal') x = np.random.random((1, 3)) y = np.random.random((1, 3, 3)) model.train_on_batch(x, y) _, fname = tempfile.mkstemp('.h5') model.save(fname) custom_objects = { 'TestOptimizer': lambda **kwargs: hvd.DistributedOptimizer( TestOptimizer(**kwargs)) } new_model = hvd.load_model(fname, custom_objects=custom_objects) new_opt = new_model.optimizer os.remove(fname) self.assertEqual(type(new_opt).__module__, 'horovod.keras') self.assertEqual(type(new_opt).__name__, 'TestOptimizer') self.assertEqual(K.get_value(opt.lr), K.get_value(new_opt.lr)) self.assertEqual(len(opt.get_weights()), len(new_opt.get_weights())) for weights, new_weights in zip(opt.get_weights(), new_opt.get_weights()): self.assertListEqual(weights.tolist(), new_weights.tolist())
def on_batch_begin(self, batch, logs={}): # print self.batch_num for i in xrange(len(self.batch_point)): if self.batch_num < self.batch_point[i]: break elif self.batch_num == self.batch_point[i]: if i < len(self.lr): K.set_value(self.model.optimizer.lr, self.lr[i]) print 'current lr:', K.get_value(self.model.optimizer.lr) self.batch_num += 1
def on_epoch_end(self, epoch, logs={}): report = {} report['epoch'] = epoch for k, v in logs.items(): report[k] = v if hasattr(self.model.optimizer, 'lr'): report['lr'] = K.get_value(self.model.optimizer.lr).tolist() self.task.push('output.training_history', report) self.task.set('output.elapsed_time', "%.2fs"%self.elapsed_time) self.task.set('output.last_epoch_update_time', datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def test_automatic_lr_scheduler(): optimizer = Adam(lr=0.1) lr_scheduler = AutomaticLearningRateScheduler(optimizer, 'loss') lr_scheduler.on_train_begin() for i in range(3): lr_scheduler.on_epoch_begin(i) lr_scheduler.on_batch_end(0, {'loss': 1/(i+1)}) lr_scheduler.on_epoch_end(i) o = 3 assert np.allclose(K.get_value(optimizer.lr), 0.1) for i in range(o, o+5): print(i) lr_scheduler.on_epoch_begin(i) lr_scheduler.on_batch_end(0, {'loss': 1}) lr_scheduler.on_epoch_end(i) print(lr_scheduler.current_best) print(lr_scheduler.current_best_epoch) assert np.allclose(K.get_value(optimizer.lr), 0.1 * lr_scheduler.factor)
def VGG_16(weights_path=None): model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=(3, 224, 224))) model.add(Convolution2D(64, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(64, 3, 3, activation="relu")) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation="relu")) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation="relu")) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation="relu")) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation="relu")) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation="relu")) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(Flatten()) model.add(Dense(4096, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(4096, activation="relu")) model.add(Dropout(0.5)) model.add(Dense(1000, activation="softmax")) if weights_path: model.load_weights(weights_path) ops = [] for layer in model.layers: if layer.__class__.__name__ in ["Convolution1D", "Convolution2D"]: original_w = K.get_value(layer.W) converted_w = convert_kernel(original_w) ops.append(tf.assign(layer.W, converted_w).op) K.get_session().run(ops) return model
def get_states(model): all_states = [] for layer in model.layers: if hasattr(layer,"states"): layer_states = [] for state in layer.states: #print(K.get_value(state)[0][0:3]) layer_states.append(K.get_value(state)) all_states.append(layer_states) # print(all_states) return all_states
def show_batch_normalization_layer(layer): """Serialize batch normalization layer to dict""" layer_axis = None if isinstance(layer.axis, int): layer_axis = layer.axis else: assert len(layer.axis) == 1 layer_axis = layer.axis[0] assert layer_axis == -1 or layer_axis + 1 == len(layer.input_shape) moving_mean = K.get_value(layer.moving_mean) moving_variance = K.get_value(layer.moving_variance) result = {} result['moving_mean'] = encode_floats(moving_mean) result['moving_variance'] = encode_floats(moving_variance) if layer.center: beta = K.get_value(layer.beta) result['beta'] = encode_floats(beta) if layer.scale: gamma = K.get_value(layer.gamma) result['gamma'] = encode_floats(gamma) return result
def _runner(init, shape, target_mean=None, target_std=None, target_max=None, target_min=None): variable = init(shape) output = K.get_value(variable) lim = 1e-2 if target_std is not None: assert abs(output.std() - target_std) < lim if target_mean is not None: assert abs(output.mean() - target_mean) < lim if target_max is not None: assert abs(output.max() - target_max) < lim if target_min is not None: assert abs(output.min() - target_min) < lim
def on_batch_end(self, batch, logs=None): if self.current_epoch_ > 1: return if self.use_validation_set: X, Y = self.validation_data[0], self.validation_data[1] # use 5 random batches from test set for fast approximate of loss num_samples = self.batch_size * self.validation_sample_rate if num_samples > X.shape[0]: num_samples = X.shape[0] idx = np.random.choice(X.shape[0], num_samples, replace=False) x = X[idx] y = Y[idx] values = self.model.evaluate(x, y, batch_size=self.batch_size, verbose=False) loss = values[0] else: loss = logs['loss'] # smooth the loss value and bias correct running_loss = self.loss_smoothing_beta * loss + ( 1. - self.loss_smoothing_beta) * loss running_loss = running_loss / ( 1. - self.loss_smoothing_beta ** self.current_batch_) # stop logging if loss is too large if self.current_batch_ > 1 and self.stopping_criterion_factor is not None and ( running_loss > self.stopping_criterion_factor * self.best_loss_): if self.verbose: print(" - LRFinder: Skipping iteration since loss is %d times as large as best loss (%0.4f)" % (self.stopping_criterion_factor, self.best_loss_)) return if running_loss < self.best_loss_ or self.current_batch_ == 1: self.best_loss_ = running_loss current_lr = K.get_value(self.model.optimizer.lr) self.history.setdefault('running_loss_', []).append(running_loss) if self.lr_scale == 'exp': self.history.setdefault('log_lrs', []).append(np.log10(current_lr)) else: self.history.setdefault('log_lrs', []).append(current_lr) # compute the lr for the next batch and update the optimizer lr if self.lr_scale == 'exp': current_lr *= self.lr_multiplier_ else: current_lr = self.lr_multiplier_[self.current_batch_ - 1] K.set_value(self.model.optimizer.lr, current_lr) # save the other metrics as well for k, v in logs.items(): self.history.setdefault(k, []).append(v) if self.verbose: if self.use_validation_set: print(" - LRFinder: val_loss: %1.4f - lr = %1.8f " % (values[0], current_lr)) else: print(" - LRFinder: lr = %1.8f " % current_lr)
def on_epoch_begin(self, epoch, logs={}): lr = float(K.get_value(self.model.optimizer.lr)) print('VAE lr:', lr) print('VAE adam:', self.model.optimizer.get_config())
def train(BATCH_SIZE, ENC_WEIGHTS, DEC_WEIGHTS): print ("Loading data definitions...") frames_source = hkl.load(os.path.join(DATA_DIR, 'sources_train_208.hkl')) videos_list = get_video_lists(frames_source=frames_source, stride=4) n_videos = videos_list.shape[0] # Setup test val_frames_source = hkl.load(os.path.join(VAL_DATA_DIR, 'sources_val_208.hkl')) val_videos_list = get_video_lists(frames_source=val_frames_source, stride=(int(VIDEO_LENGTH/2))) n_val_videos = val_videos_list.shape[0] if RAM_DECIMATE: frames = load_to_RAM(frames_source=frames_source) if SHUFFLE: # Shuffle images to aid generalization videos_list = np.random.permutation(videos_list) # Build the Spatio-temporal Autoencoder print ("Creating models...") encoder = encoder_model() print (encoder.summary()) decoder = decoder_model() autoencoder = autoencoder_model(encoder, decoder) autoencoder.compile(loss="mean_squared_error", optimizer=OPTIM_A) run_utilities(encoder, decoder, autoencoder, ENC_WEIGHTS, DEC_WEIGHTS) NB_ITERATIONS = int(n_videos/BATCH_SIZE) # NB_ITERATIONS = 5 NB_VAL_ITERATIONS = int(n_val_videos / BATCH_SIZE) # Setup TensorBoard Callback TC = tb_callback.TensorBoard(log_dir=TF_LOG_DIR, histogram_freq=0, write_graph=False, write_images=False) LRS = lrs_callback.LearningRateScheduler(schedule=schedule) LRS.set_model(autoencoder) print ("Beginning Training...") # Begin Training for epoch in range(1, NB_EPOCHS_AUTOENCODER): if epoch == 21: autoencoder.compile(loss="mean_absolute_error", optimizer=OPTIM_B) print("\n\nEpoch ", epoch) loss = [] val_loss = [] # Set learning rate every epoch LRS.on_epoch_begin(epoch=epoch) lr = K.get_value(autoencoder.optimizer.lr) print ("Learning rate: " + str(lr)) for index in range(NB_ITERATIONS): # Train Autoencoder if RAM_DECIMATE: X = load_X_RAM(videos_list, index, frames) else: X = load_X(videos_list, index, DATA_DIR, IMG_SIZE) X_train = np.flip(X[:, 0 : int(VIDEO_LENGTH/2)], axis=1) y_train = X[:, int(VIDEO_LENGTH/2) :] loss.append(autoencoder.train_on_batch(X_train, y_train)) arrow = int(index / (NB_ITERATIONS / 40)) stdout.write("\rIter: " + str(index) + "/" + str(NB_ITERATIONS-1) + " " + "loss: " + str(loss[len(loss)-1]) + "\t [" + "{0}>".format("="*(arrow))) stdout.flush() if SAVE_GENERATED_IMAGES: # Save generated images to file predicted_images = autoencoder.predict(X_train, verbose=0) voila = np.concatenate((X_train, y_train), axis=1) truth_seq = arrange_images(voila) pred_seq = arrange_images(np.concatenate((X_train, predicted_images), axis=1)) truth_seq = truth_seq * 127.5 + 127.5 pred_seq = pred_seq * 127.5 + 127.5 if epoch == 1: cv2.imwrite(os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + "_truth.png"), truth_seq) cv2.imwrite(os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + "_pred.png"), pred_seq) # Run over test data print ('') for index in range(NB_VAL_ITERATIONS): X = load_X(val_videos_list, index, VAL_DATA_DIR, IMG_SIZE) X_val = np.flip(X[:, 0: int(VIDEO_LENGTH / 2)], axis=1) y_val = X[:, int(VIDEO_LENGTH / 2):] val_loss.append(autoencoder.test_on_batch(X_val, y_val)) arrow = int(index / (NB_VAL_ITERATIONS / 40)) stdout.write("\rIter: " + str(index) + "/" + str(NB_VAL_ITERATIONS - 1) + " " + "test_loss: " + str(val_loss[len(val_loss) - 1]) + "\t [" + "{0}>".format("=" * (arrow))) stdout.flush() # then after each epoch/iteration avg_loss = sum(loss)/len(loss) avg_val_loss = sum(val_loss) / len(val_loss) logs = {'loss': avg_loss, 'val_loss': avg_val_loss} TC.on_epoch_end(epoch, logs) # Log the losses with open(os.path.join(LOG_DIR, 'losses_gen.json'), 'a') as log_file: log_file.write("{\"epoch\":%d, \"train_loss\":%f, \"val_loss\":%f}\n" % (epoch, avg_loss, avg_val_loss)) print("\nAvg train loss: " + str(avg_loss) + " Avg val loss: " + str(avg_val_loss)) # Save model weights per epoch to file if epoch>15 and epoch<21: encoder.save_weights(os.path.join(CHECKPOINT_DIR, 'encoder_epoch_' + str(epoch) + '.h5'), True) decoder.save_weights(os.path.join(CHECKPOINT_DIR, 'decoder_epoch_' + str(epoch) + '.h5'), True) if epoch>25: encoder.save_weights(os.path.join(CHECKPOINT_DIR, 'encoder_epoch_' + str(epoch) + '.h5'), True) decoder.save_weights(os.path.join(CHECKPOINT_DIR, 'decoder_epoch_' + str(epoch) + '.h5'), True)
def get_tensor_value(x): if isinstance(x, tf.Tensor): return K.get_value(x) return x
float_data[i, :] = values # Train Set float_data = shuffel(float_data) #Nrmalisation train_data float_data[:, :-1] = dp.zero_mean_normalization(float_data[:, :-1]) data = float_data[:, :-1] train_data = data[150:] label_data = float_data[150:, -1] #validation data # MCAR train_data_voll = K.dropout(train_data, 0.0001, None, None) train_data_voll = K.get_value(train_data_voll) train_data_20_MCAR = K.dropout(train_data, 0.2, None, None) train_data_20_MCAR = K.get_value(train_data_20_MCAR) train_data_20_MCAR *= 0.8 train_data_40_MCAR = K.dropout(train_data, 0.4, None, None) train_data_40_MCAR = K.get_value(train_data_40_MCAR) train_data_40_MCAR *= 0.6 #MAR sort_float_data = sorted(float_data[150:], key=lambda x: x[1]) sort_float_data = np.array(sort_float_data) sort_train_data = sort_float_data[:, :-1] label_data_MAR = sort_float_data[:, -1]
def main(args): # set the image parameters img_rows = args.img_rows img_cols = args.img_cols img_dim_color = args.img_channels # mix_prop = 1.0 # set the value of the mixing proportion ############################################################################################################# ################################## DEFINING MODEL ########################################################## ############################################################################################################## model_alex = cnn_hybrid_color_single(img_rows, img_cols, img_dim_color) # load the model # model_final = Model(model_alex.input, model_alex.output) # specify the input and output of the model model_final = model_alex print(model_final.summary()) # print the model summary plot_model(model_final, to_file='./NIN_hybrid_bin_resnet_1x1-class', show_shapes=True) # save the model summary as a png file lr = args.learning_rate # set the learning rate # set the optimizer optimizer = SGD(lr=lr, decay=1e-6, momentum=0.9) # model compilation model_final.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) # print the value of the learning rate print(K.get_value(optimizer.lr)) # -------------------------------------------------- ############################################################################################################# ########################## GETTING TRAINING DATA AND TESTING DATA ########################################## ############################################################################################################## # get the training data by calling the pairs function # read the training data train_pairs_r, training_data_r, training_label_r = read_pairs( args.tr_img_lab_r) train_pairs_l, training_data_l, training_label_l = read_pairs( args.tr_img_lab_l) assert len(training_data_r) == len(training_data_l) # combine the left and right image in the training data to make a X x Y x 6 tensor training_data = [] for i in range(len(training_data_r)): # define the stereo pair stereo_pair = [training_data_r[i], training_data_l[i]] training_data.append(stereo_pair) batch_num = 0 # initialize the live samples and fake samples live_samples_ub = 0 attack_samples_ub = 0 live_samples = [] live_labels = [] attack_samples = [] attack_labels = [] # separate the live samples and fake samples to balance the both classes, i.e. live class and fake class assert len(training_label_r) == len(training_label_l) for i in range(len(training_data)): if training_label_r[i] == 0: live_samples.append(training_data[i]) live_labels.append(training_label_r[i]) live_samples_ub += 1 elif (training_label_r[i] == 1) | (training_label_r[i] == 2) | ( training_label_r[i] == 4): # protocol_2 attack_samples.append(training_data[i]) attack_labels.append(training_label_r[i]) attack_samples_ub += 1 print("Live samples are %g ,\t attack samples are %g" % (live_samples_ub, attack_samples_ub)) # compute the difference; the live samples are always less than the fake samples in our case diff = 0 if live_samples_ub < attack_samples_ub: # compute the ratio diff = np.int(attack_samples_ub / live_samples_ub) print("The difference is :%g " % (diff)) else: ValueError("The fake samples are less than then live samples") # number of times the dataset has to be copied: live_samples_b = live_samples live_labels_b = live_labels for i in range(diff - 1): # print("length before balancing: %g" %len(live_samples_b)) sl_copy = live_samples.copy() ll_copy = live_labels.copy() live_samples_b = live_samples_b + sl_copy live_labels_b = live_labels_b + ll_copy # print("length after balancing: %g" % len(live_samples_b)) # balanced data training_data_balanced = live_samples_b + attack_samples training_label_balanced = live_labels_b + attack_labels print("Balanced data samples: %g" % len(training_data_balanced)) # get the length of the training data len_tr = len(training_data_balanced) # get the number equal to the length of the training data indices_tr = np.arange(len_tr) np.random.shuffle(indices_tr) # initialize the image counter images_read = 0 train_img_data_r = [] train_img_data_l = [] for i in indices_tr: if training_label_balanced[i] > 0: training_label_balanced[i] = 1 train_img_data_r.append( [training_data_balanced[i][0], training_label_balanced[i]]) # read the right image train_img_data_l.append( [training_data_balanced[i][1], training_label_balanced[i]]) # read the left image # print(training_data_balanced[i][1]) # cv2.imshow('img1', cv2.imread(training_data_balanced[i][0])) # cv2.waitKey() # cv2.imshow('img2', cv2.imread(training_data_balanced[i][1])) # cv2.waitKey() images_read += 1 sys.stdout.write('train images read = {0}\r'.format(images_read)) sys.stdout.flush() ############################################################################################################ # read the test data test_pairs, test_data_r, test_labels_r = read_pairs(args.tst_img_lab_r) test_pairs, test_data_l, test_labels_l = read_pairs(args.tst_img_lab_l) assert len(test_data_r) == len(test_data_l) # combine the left and right image in the training data to make a X x Y x 6 tensor test_data = [] for i in range(len(test_data_r)): # define the stereo pair stereo_pair_t = [test_data_r[i], test_data_l[i]] test_data.append(stereo_pair_t) test_labels = test_labels_r images_read = 0 # get the length of the training data len_test = len(test_data) # get the number equal to the length of the training data indices_test = np.arange(len_test) test_img_data_r = [] test_img_data_l = [] for i in indices_test: if test_labels[i] > 0: test_labels[i] = 1 test_img_data_r.append([test_data[i][0], test_labels[i]]) # read the right test image test_img_data_l.append([test_data[i][1], test_labels[i]]) # red the left test image images_read += 1 sys.stdout.write('test images read = {0}\r'.format(images_read)) sys.stdout.flush() ##################################################################################################### # make all the data in panda data frame format train_df_r = pd.DataFrame(train_img_data_r) train_df_r.columns = ['id_r', 'label'] train_df_l = pd.DataFrame(train_img_data_l) train_df_l.columns = ['id_l', 'label'] test_df_r = pd.DataFrame(test_img_data_r) test_df_r.columns = ['id_r', 'label'] test_df_l = pd.DataFrame(test_img_data_l) test_df_l.columns = ['id_l', 'label'] ########################################################################################################333 datagen = image.ImageDataGenerator() train_generator_r = datagen.flow_from_dataframe( dataframe=train_df_r, directory=None, x_col='id_r', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=True, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest', drop_duplicates=False) train_generator_l = datagen.flow_from_dataframe( dataframe=train_df_l, directory=None, x_col='id_l', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=True, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest', drop_duplicates=False) test_datagen = image.ImageDataGenerator() test_generator_r = test_datagen.flow_from_dataframe( dataframe=test_df_r, directory=None, x_col='id_r', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=False, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest') test_generator_l = test_datagen.flow_from_dataframe( dataframe=test_df_l, directory=None, x_col='id_l', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=False, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest') ############################################################################################################# batch_num = 0 while batch_num < args.max_epochs: start_time = time.time() # initialize the clock acc = [] loss = [] sub_count = 0 total_batch = train_generator_r.n // train_generator_r.batch_size for i in range(train_generator_r.n // train_generator_r.batch_size): x1, y1 = next(train_generator_r) x2, y2 = next(train_generator_l) # only for DP-3D for comparison # disparity_final = [] # # for j in range(x1.shape[0]): # img1 = np.asarray(x1[j]) # # img1 = cv2.resize(img1, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # img2 = np.asarray(x2[j]) # # img2 = cv2.resize(img2, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # # disparity = cv2.subtract(img1, img2) # # der_k = np.asarray([[1.0, 2.0, 1.0], # [0.0, 0.0, 0.0], # [-1.0, -2.0, -1.0]]) # # der = cv2.filter2D(img1, -1, kernel=der_k) # # disparity_f = disparity / (der + 0.005) # # disparity_final.append(disparity_f) # # # disparity_final = np.asarray(disparity_final).astype('float32') # disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'),axis=-1) x1 = x1.astype('float32') / 255 x2 = x2.astype('float32') / 255 y = y1 tr_acc1 = model_final.fit([x1, x2], y, epochs=1, verbose=0) acc.append(tr_acc1.history['acc'][0]) loss.append(tr_acc1.history['loss'][0]) sub_count += 1 sys.stdout.write('batch_count = {0} of {1} \r'.format( sub_count, total_batch)) sys.stdout.flush() train_acc = np.sum(np.asarray(acc)) * 100 / ( train_generator_r.n // train_generator_r.batch_size) train_loss = np.sum(np.asarray(loss)) * 100 / ( train_generator_r.n // train_generator_r.batch_size) print('training_acc: {0} \t training_loss: {1}'.format( train_acc, train_loss)) print( '______________________________________________________________________' ) print('Running the evaluations') test_acc = [] test_loss = [] sub_count = 0 for i in range(test_generator_r.n // test_generator_r.batch_size): x1, y1 = next(test_generator_r) x2, y2 = next(test_generator_l) # only for DP-3D for comparison # disparity_final = [] # # for j in range(x1.shape[0]): # img1 = np.asarray(x1[j]) # # img1 = cv2.resize(img1, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # img2 = np.asarray(x2[j]) # # img2 = cv2.resize(img2, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # # disparity = cv2.subtract(img1, img2) # # der_k = np.asarray([[1.0, 2.0, 1.0], # [0.0, 0.0, 0.0], # [-1.0, -2.0, -1.0]]) # # der = cv2.filter2D(img1, -1, kernel=der_k) # # disparity_f = disparity / (der + 0.005) # # disparity_final.append(disparity_f) # # # disparity_final = np.asarray(disparity_final).astype('float32') # disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'), axis=-1) x1 = x1.astype('float32') / 255 x2 = x2.astype('float32') / 255 y = y1 tst_loss, tst_acc1 = model_final.evaluate([x1, x2], y, verbose=0) test_acc.append(tst_acc1) test_loss.append(tst_loss) sub_count += 1 sys.stdout.write('epoch_count = {0}\r'.format(sub_count)) sys.stdout.flush() test_acc = np.sum(np.asarray(test_acc)) * 100 / ( test_generator_r.n // test_generator_r.batch_size) test_loss = np.sum(np.asarray(test_loss)) * 100 / ( test_generator_r.n // test_generator_r.batch_size) print('test_acc: {0} \t test_loss: {1}'.format(test_acc, test_loss)) batch_num += 1 # ********************************************************************************************** # learning rate schedule update: if learning is done using a single learning give the batch_num below a # high value if (batch_num == 3) | (batch_num == 5) | (batch_num == 7): lr = 0.1 * lr K.set_value(optimizer.lr, lr) print(K.get_value(optimizer.lr)) # ************************************************************************************************ # ----------------------------------------------------------------------------------------------- end_time = time.time() - start_time print("Total time taken %f :" % end_time) model_final.save_weights( '/home/yaurehman2/Documents/stereo_face_liveness/stereo_ckpt/Conventional/' + 'dual_grayscale_input_revised_protocol_2_' + str(args.max_epochs) + '.h5')
def on_epoch_end(self, epoch, logs={}): self.seen += logs.get('size', 0) #print([l.name for l in self.model.layers]) if epoch % self.display == 0: metrics_log = '' for k in self.params['metrics']: if k in logs: val = logs[k] if abs(val) > 1e-3: metrics_log += ' - %s: %.4f' % (k, val) else: metrics_log += ' - %s: %.4e' % (k, val) #weight = self.model.loss.keywords['weight'] if(self.is_VAE): weight = K.get_value(self.model.loss_weights['decoder_for_kl']) #print(self.model.get_layer('sample_z').ouput.values()) inputTensor=[self.model.get_layer('x_true').input] #inputTensor=[self.model.get_layer('enc_x_true').input,self.model.get_layer('enc_cond').input] n_cond_pre=0 n_embs_input=0 for l in self.model.layers: if('cond_pre' in l.name ): inputTensor.append(self.model.get_layer(l.name).input) n_cond_pre=n_cond_pre+1 for l in self.model.layers: if('emb_input' in l.name ): inputTensor.append(self.model.get_layer(l.name).input) n_embs_input=n_embs_input+1 print(l.name ) x_inputs =self.x_train_data inputsY=x_inputs[0] if(n_cond_pre>=1): if(n_cond_pre==1): cond_pre=x_inputs[1] else: cond_pre=x_inputs[1:(1+n_cond_pre)] if(n_embs_input>=1): emb_inputs=x_inputs[(1+n_cond_pre):] embModel=self.model.get_layer('embedding_enc') emb_ouputs = embModel.predict(emb_inputs) cond = np.concatenate((cond_pre, emb_ouputs), axis=1) input_encoder = [inputsY,cond] else: input_encoder = [inputsY,cond_pre] elif(n_embs_input>=1): emb_inputs=x_inputs[1:] embModel=self.model.get_layer('embedding_enc') emb_ouputs = embModel.predict(emb_inputs) #emb_ouputs=np.squeeze(emb_ouputs, axis=0) input_encoder = [inputsY,emb_ouputs] else: input_encoder=[inputsY] self.response_model=self.model.get_layer('encoder') responses=self.response_model.predict(input_encoder) if(self.is_VAE): responses=responses[0] #responses=self.model.encoder.predict(self.x_train_data) print(np.sum(np.abs(responses),axis=0)) predictFeaturesInLatentSPace(self.x_conso,self.calendar_info,responses,k=5) valLoss=logs.get('val_loss') if(self.is_VAE): print('{} Epochs ... {} val_loss {} ... lambda Loss {}'.format(epoch, metrics_log,valLoss,weight)) else: print('{} Epochs ... {}'.format(epoch, metrics_log))
model2 = Model(inputs=new_input, outputs=[new_out_softmax]) model2.summary() input("Press Enter to continue.") # Training: try: model2.fit([imgs_train, bots_train], lab_train_ohe, batch_size=50, epochs=110, verbose=1, validation_data=([imgs_val, bots_val], lab_val_ohe), callbacks=[RegScheduler(beta=beta, epoch_count=epoch_count)]) score, accuracy = model2.evaluate([imgs_test, bots_test], lab_test_ohe, batch_size=100, verbose=0) print('Test score:', score) print('Test accuracy:', accuracy) epc_count= int(K.get_value(epoch_count))+1 file_name = 'model_exp_'+str(epc_count)+'.h5' model_path_save = os.path.join(model_dir, file_name) model.save(model_path_save) # Layers sizes input("Press Enter to continue.") bot_lay_size = bots_train.shape[1] n_train_imgs = imgs_train.shape[0] n_test_imgs = imgs_test.shape[0] n_classes = lab_train_ohe.shape[1] print("bot lay size:", bot_lay_size) print("train_imgs", n_train_imgs) print('test imgs', n_test_imgs) print("classes", n_classes)
def on_epoch_end(self, epoch, logs={}): max_epoch= 90 power = 4 stop = 0 K.set_value(self.beta, ((1-(epoch/max_epoch)) ** power ) * (1-stop) + stop ) print('---current beta: %.3f' % K.get_value(beta))
def scheduler(epoch): if epoch % 20 == 0 and epoch != 0: lr = K.get_value(model.optimizer.lr) K.set_value(model.optimizer.lr, lr * 0.1) print("lr changed to {}".format(lr * 0.1)) return K.get_value(model.optimizer.lr)
3, 2, (3, 4, 6), (64, 128, 256, 512), reg=0.0005, dataset="tiny_imagenet") opt = SGD(lr=1e-1, momentum=0.9) model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"]) # otherwise, load the checkpoint from disk else: print("[INFO] loading {}...".format(args["model"])) model = load_model(args["model"]) # update the learning rate print("[INFO] old learning rate: {}".format(K.get_value( model.optimizer.lr))) K.set_value(model.optimizer.lr, 1e-3) print("[INFO] new learning rate: {}".format(K.get_value( model.optimizer.lr))) # construct the set of callbacks figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())]) jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())]) fname = os.path.sep.join( [args["weights"], "weights-{epoch:03d}-{val_acc:.4f}.hdf5"]) checkpoint = ModelCheckpoint(fname, monitor="val_acc", mode="max", save_best_only=True, verbose=1) callbacks = [
def test_dynamic_loss_scaling(self, strategy_fn, pass_loss_scale_to_policy=False, get_config=False, use_v1_loss_scale_optimizer=False): strategy = strategy_fn() initial_loss_scale = 2. batch_size = 4 expected_gradient = backend.variable([initial_loss_scale / batch_size], dtype=tf.float16) # If this variable is set to True, the model below will have NaN gradients have_nan_gradients = backend.variable(False, dtype=tf.bool) with strategy.scope(): opt = gradient_descent.SGD(1.) if pass_loss_scale_to_policy: loss_scale = tf.mixed_precision.experimental.DynamicLossScale( initial_loss_scale=initial_loss_scale, increment_period=2) p = policy.PolicyV1('mixed_float16', loss_scale=loss_scale) elif use_v1_loss_scale_optimizer: loss_scale = tf.mixed_precision.experimental.DynamicLossScale( initial_loss_scale=initial_loss_scale, increment_period=2) p = policy.Policy('mixed_float16') opt = loss_scale_optimizer.LossScaleOptimizerV1( opt, loss_scale) else: p = policy.Policy('mixed_float16') opt = loss_scale_optimizer.LossScaleOptimizer( opt, initial_scale=initial_loss_scale, dynamic_growth_steps=2) with policy.policy_scope(p): x = layers.Input(shape=(1, ), batch_size=batch_size, dtype=tf.float16) layer = mp_test_util.MultiplyLayer(assert_type=tf.float16) y = layer(x) identity_with_nan_grads = ( mp_test_util.create_identity_with_nan_gradients_fn( have_nan_gradients)) y = core.Lambda(identity_with_nan_grads)(y) identity_with_grad_check_fn = ( mp_test_util.create_identity_with_grad_check_fn( expected_dtype=tf.float16, expected_gradient=expected_gradient)) y = core.Lambda(identity_with_grad_check_fn)(y) model = models.Model(inputs=x, outputs=y) if get_config: config = model.get_config() model = model.__class__.from_config( config, custom_objects={ 'MultiplyLayer': mp_test_util.MultiplyLayer }) (layer, ) = ( layer for layer in model.layers if isinstance(layer, mp_test_util.MultiplyLayer)) def loss_fn(y_true, y_pred): del y_true return tf.reduce_mean(y_pred) model.compile(opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly()) self.assertEqual(backend.eval(layer.v), 1) x = np.ones((batch_size, 1)) y = np.ones((batch_size, 1)) dataset = tf.data.Dataset.from_tensor_slices((x, y)).batch(batch_size) model.fit(dataset) # The variables starts with 1 and has a gradient of 1, so will go down by 1 # each step. self.assertEqual(backend.eval(layer.v), 0) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -1) # There have been two steps without NaNs, so the loss scale will double backend.set_value(expected_gradient, backend.get_value(expected_gradient * 2)) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -2) # Next test with NaN gradients. backend.set_value(have_nan_gradients, True) model.fit(dataset) # Variable should not be updated self.assertEqual(backend.eval(layer.v), -2) # Test with finite gradients again backend.set_value(have_nan_gradients, False) # The loss scale will be halved due to the NaNs, so the gradient will also # be halved backend.set_value(expected_gradient, backend.get_value(expected_gradient / 2)) model.fit(dataset) self.assertEqual(backend.eval(layer.v), -3)
def on_epoch_end(self, epochs, logs={}): lr = K.get_value(self.model.optimizer.lr) print('\nLearningRate:{:.6f}'.format(lr))
model = keras.models.load_model(path_to_model, custom_objects={'psnr': psnr}) # Check its architecture model.summary() # Do the prediction pred = model.predict(test_x) # Print PREDICTION print("--PREDICTION--") print(pred) # Calculate PSNR between ORIGINAL and PREDICTION print("--PSNR--") psnr_array = K.get_value(psnr(test_y, pred)) print(psnr_array) fig = plt.figure(figsize=(50, 50), facecolor="w") plt.subplot(1, 3, 1) plt.title("Original", fontsize=80) plt.tight_layout() plt.imshow(test_y[N_show, :, :]) plt.subplot(1, 3, 2) plt.title("Low resolution", fontsize=80) plt.tight_layout() plt.imshow(test_x[N_show, :, :]) plt.subplot(1, 3, 3)
def learn(self, model, model_file=None, start_from_scratch=False, reference_states=None, input_shape=None): q_progress = {'q_values': [], 'epochs': []} tensorboard = TensorBoard(log_dir="logs/{}".format(time())) if model_file is not None and not start_from_scratch: model.load_weights(model_file) # TODO: find alternative way for calculating reference-q-values if reference_states is not None: q_progress['q_values'].append( self.get_avg_max_q(model, reference_states=reference_states, shape=input_shape)) q_progress['epochs'].append(0) # Train for e in range(self.q_learning_epochs): all_inputs = None all_targets = None all_q_deltas = [] for el in self.experience_list: # the Q-learning magic happens here... inputs, targets, q_delta = el.get_batch( model, single_actions=self.single_actions, p=0.25) if input_shape is not None: inputs.reshape((inputs.shape[0], ) + input_shape) if (all_inputs is None): all_inputs = inputs all_targets = targets else: all_inputs = np.concatenate((all_inputs, inputs), axis=0) all_targets = np.concatenate((all_targets, targets), axis=0) all_q_deltas += q_delta logging.debug("q_deltas: N: %d, min %f, max %f, avg %f" % (len(all_q_deltas), min(all_q_deltas), max(all_q_deltas), statistics.mean(all_q_deltas))) # adapt model X_train, X_test, y_train, y_test = train_test_split( all_inputs, all_targets, test_size=0.2) #Xrandom_state=42) _lr = K.get_value(model.optimizer.lr) logging.debug(" Current learning rate (before fit):" + str(_lr)) #K.set_value(model.optimizer.lr, _lr/10.) reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1) #model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=self.fixed_learning_epochs, batch_size=self.batch_size, callbacks=[tensorboard, reduce_lr]) model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=self.fixed_learning_epochs, batch_size=self.batch_size, callbacks=[tensorboard]) _lr = K.get_value(model.optimizer.lr) logging.debug(" Current learning rate (after fit):" + str(_lr)) scores = model.evaluate(X_test, y_test, verbose=0) print(e, scores) print("Baseline Error: %.2f%%" % (100 - scores * 100)) loss = scores print("Epoch {:03d}/{:03d} | Loss {:.4f}".format( e, self.q_learning_epochs, loss)) if reference_states is not None: q_progress['q_values'].append( self.get_avg_max_q(model, reference_states=reference_states, shape=input_shape)) prev_epochs = q_progress['epochs'][len(q_progress['epochs']) - 1] q_progress['epochs'].append(prev_epochs + len(self.experience_list)) # Save trained model weights and architecture, this will be used by the visualization code if model_file is not None: model.save_weights(model_file, overwrite=True) return q_progress
def get_config(self): config = {'lr': float(K.get_value(self.lr))} base_config = super(SGDCust, self).get_config() return dict(list(base_config.items()) + list(config.items()))
def get_config(self): config = {'death_rate': K.get_value(self.death_rate)} base_config = super().get_config() return dict(list(base_config.items()) + list(config.items()))
def m_global(self): return K.get_value(self.m_global_var)
def build(self, input_shape): self.input_spec = [InputSpec(shape=input_shape)] self.input_dim = input_shape[2] if self.stateful: self.reset_states() else: self.states = [None] self.W = self.init((self.input_dim, (2 + (not self.coupling)) * self.output_dim), name='{}_W'.format(self.name)) self.Us = [self.inner_init( (self.output_dim, (2 + (not self.coupling)) * self.output_dim), name='%s_%d_U' % (self.name, i)) for i in xrange(self.depth)] bias_init_value = K.get_value(self.bias_init((self.output_dim,))) b = [np.zeros(self.output_dim), np.copy(bias_init_value)] if not self.coupling: b.append(np.copy(bias_init_value)) self.bs = [K.variable(np.hstack(b), name='%s_%d_b' % (self.name, i)) for i in xrange(self.depth)] self.trainable_weights = [self.W] + self.Us + self.bs if self.mi: self.mi_params = [multiplicative_integration_init( ((2 + (not self.coupling)) * self.output_dim,), name='%s_%d' % (self.name, i), has_input=(i == 0)) for i in xrange(self.depth)] for p in self.mi_params: if type(p) in {list, tuple}: self.trainable_weights += p else: self.trainable_weights += [p] if self.has_layer_norm: self.ln_weights = [] ln_names = ['h', 't', 'c'] for l in xrange(self.depth): ln_gains = [self.ln_gain_init( (self.output_dim,), name='%s_%d_ln_gain_%s' % (self.name, l, ln_names[i])) for i in xrange(1)] ln_biases = [self.ln_bias_init( (self.output_dim,), name='%s_%d_ln_bias_%s' % (self.name, l, ln_names[i])) for i in xrange(1)] self.ln_weights.append([ln_gains, ln_biases]) self.trainable_weights += ln_gains + ln_biases self.regularizers = [] if self.W_regularizer: self.W_regularizer.set_param(self.W) self.regularizers.append(self.W_regularizer) if self.U_regularizer: self.U_regularizer.set_param(self.U) self.regularizers.append(self.U_regularizer) if self.b_regularizer: self.b_regularizer.set_param(self.b) self.regularizers.append(self.b_regularizer) if self.initial_weights is not None: self.set_weights(self.initial_weights) del self.initial_weights
def k(self): return K.get_value(self.k_var)
def train(BATCH_SIZE, ENC_WEIGHTS, DEC_WEIGHTS): print("Loading data...") frames_source = hkl.load(os.path.join(DATA_DIR, 'sources_train_128.hkl')) # Build video progressions videos_list = [] start_frame_index = 1 end_frame_index = VIDEO_LENGTH + 1 while (end_frame_index <= len(frames_source)): frame_list = frames_source[start_frame_index:end_frame_index] if (len(set(frame_list)) == 1): videos_list.append(range(start_frame_index, end_frame_index)) start_frame_index = start_frame_index + 1 end_frame_index = end_frame_index + 1 else: start_frame_index = end_frame_index - 1 end_frame_index = start_frame_index + VIDEO_LENGTH videos_list = np.asarray(videos_list, dtype=np.int32) n_videos = videos_list.shape[0] if SHUFFLE: # Shuffle images to aid generalization videos_list = np.random.permutation(videos_list) # Build the Spatio-temporal Autoencoder print("Creating models...") encoder = encoder_model() decoder = decoder_model() autoencoder = autoencoder_model(encoder, decoder) run_utilities(encoder, decoder, autoencoder, ENC_WEIGHTS, DEC_WEIGHTS) autoencoder.compile(loss='mean_squared_error', optimizer=OPTIM) NB_ITERATIONS = int(n_videos / BATCH_SIZE) # Setup TensorBoard Callback TC = tb_callback.TensorBoard(log_dir=TF_LOG_DIR, histogram_freq=0, write_graph=False, write_images=False) LRS = lrs_callback.LearningRateScheduler(schedule=schedule) LRS.set_model(autoencoder) print("Beginning Training...") # Begin Training for epoch in range(NB_EPOCHS): print("\n\nEpoch ", epoch) loss = [] # Set learning rate every epoch LRS.on_epoch_begin(epoch=epoch) lr = K.get_value(autoencoder.optimizer.lr) print("Learning rate: " + str(lr)) for index in range(NB_ITERATIONS): # Train Autoencoder X = load_X(videos_list, index, DATA_DIR) # X_train = X[:, 0 : int(VIDEO_LENGTH/2)] # y_train = X[:, int(VIDEO_LENGTH/2) :] # Change made to generate more output per input frame X_train = X[:, 0:10] y_train = X[:, 10:] loss.append(autoencoder.train_on_batch(X_train, y_train)) arrow = int(index / (NB_ITERATIONS / 40)) stdout.write("\rIteration: " + str(index) + "/" + str(NB_ITERATIONS - 1) + " " + "loss: " + str(loss[len(loss) - 1]) + "\t [" + "{0}>".format("=" * (arrow))) stdout.flush() if SAVE_GENERATED_IMAGES: # Save generated images to file predicted_images = autoencoder.predict(X_train, verbose=0) orig_image, truth_image, pred_image = combine_images( X_train, y_train, predicted_images) pred_image = pred_image * 127.5 + 127.5 orig_image = orig_image * 127.5 + 127.5 truth_image = truth_image * 127.5 + 127.5 if epoch == 0: cv2.imwrite( os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + "_orig.png"), orig_image) cv2.imwrite( os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + "_truth.png"), truth_image) cv2.imwrite( os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + ".png"), pred_image) # then after each epoch/iteration avg_loss = sum(loss) / len(loss) logs = {'loss': avg_loss} TC.on_epoch_end(epoch, logs) # Log the losses with open(os.path.join(LOG_DIR, 'losses.json'), 'a') as log_file: log_file.write("{\"epoch\":%d, \"d_loss\":%f};\n" % (epoch, avg_loss)) print("\nAvg loss: " + str(avg_loss)) # Save model weights per epoch to file encoder.save_weights( os.path.join(CHECKPOINT_DIR, 'encoder_epoch_' + str(epoch) + '.h5'), True) decoder.save_weights( os.path.join(CHECKPOINT_DIR, 'decoder_epoch_' + str(epoch) + '.h5'), True) # End TensorBoard Callback TC.on_train_end('_')
def main(args=None): global config from keras import backend as K # parse arguments if args is None: args = sys.argv[1:] args = parse_args(args) print('Arguments: {}'.format(args)) # create object that stores backbone information backbone = models.backbone(args.backbone) # make sure keras is the minimum required version check_keras_version() # optionally choose specific GPU if args.gpu: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu keras.backend.tensorflow_backend.set_session(get_session()) # create the generators train_generator, validation_generator = create_generators(args, backbone.preprocess_image) # create the model if args.snapshot is not None: print('Loading model, this may take a second...') model = models.load_model(args.snapshot, backbone_name=args.backbone) training_model = model anchor_params = None if 'anchor_parameters' in config: anchor_params = parse_anchor_parameters(config) prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params) else: weights = args.weights # default to imagenet if nothing else is specified if weights is None and args.imagenet_weights: weights = backbone.download_imagenet() print('Creating model, this may take a second...') model, training_model, prediction_model = create_models( backbone_retinanet=backbone.retinanet, num_classes=train_generator.num_classes(), weights=weights, args=args, multi_gpu=args.multi_gpu, freeze_backbone=args.freeze_backbone, config=config ) # print model summary print(model.summary()) print('Learning rate: {}'.format(K.get_value(model.optimizer.lr))) if args.lr > 0.0: K.set_value(model.optimizer.lr, args.lr) print('Updated learning rate: {}'.format(K.get_value(model.optimizer.lr))) # this lets the generator compute backbone layer shapes using the actual backbone model if 'vgg' in args.backbone or 'densenet' in args.backbone: train_generator.compute_shapes = make_shapes_callback(model) if validation_generator: validation_generator.compute_shapes = train_generator.compute_shapes # create the callbacks callbacks = create_callbacks( model, training_model, prediction_model, validation_generator, args, ) init_epoch = 0 try: if args.snapshot: init_epoch = int(args.snapshot.split("_")[-2]) except: pass # init_epoch = 6 print('Init epoch: {}'.format(init_epoch)) # start training training_model.fit_generator( generator=train_generator, steps_per_epoch=args.steps, epochs=args.epochs, verbose=1, callbacks=callbacks, initial_epoch=init_epoch, )
def on_epoch_begin(self, epoch, logs={}): print("Learning rate:", K.get_value(model.optimizer.lr))
eps = [] history_train_mse, history_train_angular_errors = [], [] history_val_mean_angular_errors, history_val_median_angular_errors = [], [] min_median_angular_error = float('inf') for current_epoch in range(1, EPOCHS + 1): start_time = timer() print('=' * 60) print('Epoch {}/{} started.'.format(current_epoch, EPOCHS)) # learning rate decrease if len(history_val_median_angular_errors) > PATIENCE: if np.min(history_val_median_angular_errors[-PATIENCE:] ) > history_val_median_angular_errors[-PATIENCE - 1] - MIN_DELTA: old_lr = float(K.get_value(model.optimizer.lr)) new_lr = max(old_lr * FACTOR, MIN_LR) K.set_value(model.optimizer.lr, new_lr) # learning rate increase if len(history_val_median_angular_errors) > PATIENCE * 10: if np.min( history_val_median_angular_errors[-PATIENCE * 10:] ) > history_val_median_angular_errors[-PATIENCE * 10 - 1] - MIN_DELTA: old_lr = float(K.get_value(model.optimizer.lr)) new_lr = min(old_lr * 2, MAX_LR) K.set_value(model.optimizer.lr, new_lr) print('Learning rate increased!') print('Learning rate in current epoch: {0:.2e}'.format(
def on_epoch_begin(self, epoch, logs={}): print('EPOCH:', epoch) print('\tkl_beta_weight=', K.get_value(mod_args['kl_beta_weight'])) print('\tlr=', float(K.get_value(self.model.optimizer.lr)))
# misclassification layer, feed Y y_true = Input(shape=(nclass, ), name='y_true') psi = mfom.UvZMisclassification(name='uvz_misclass')([y_true, y_pred]) # class Loss function layer out = mfom.SmoothErrorCounter(name='smooth_error_counter')(psi) # compile model model = Model(input=[y_true, feat_input], output=out) model.compile(loss=obj.mfom_eer_normalized, optimizer='Adam') model.summary() # train X, Y = generate_dataset(output_dim=nclass) hist = model.fit([Y, X], Y, nb_epoch=100, batch_size=16) # calc accuracy: we cut MFoM head, up to sigmoid output input = model.get_layer(name='main_input').output out = model.get_layer(name='output').output cut_model = Model(input=input, output=out) y_pred = cut_model.predict(X) eer_val = MT.eer(y_true=Y.flatten(), y_pred=y_pred.flatten()) print('EER: %.4f' % eer_val) # history plot, alpha and beta params m = model.get_layer('smooth_error_counter') print('alpha: ', K.get_value(m.alpha)) print('beta: ', K.get_value(m.beta)) plt.plot(hist.history['loss']) plt.show()
def on_epoch_end(self, epoch, logs=None): logs = logs or {} logs['lr'] = K.get_value(self.model.optimizer.lr)
def scheduler(epoch): K.set_value(vae.optimizer.lr, LearningRate[epoch - 1]) return float(K.get_value(vae.optimizer.lr))
def main(): """Train DeeperGoogLeNet """ # construct the argument parse and parse the arguments args = argparse.ArgumentParser() args.add_argument("-c", "--checkpoints", required=True, help="path to output checkpoint directory") args.add_argument("-m", "--model", type=str, help="path to *specific* model checkpoint to load") args.add_argument("-s", "--start-epoch", type=int, default=0, help="epoch to restart training at") args = vars(args.parse_args()) # construct the training image generator for data augmentation augmentation = ImageDataGenerator( rotation_range=18, zoom_range=0.15, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15, horizontal_flip=True, fill_mode="nearest", ) # load the RGB means for the training set means = json.loads(open(config.DATASET_MEAN).read()) # initialize the image preprocessors simple_preprocessor = SimplePreprocessor(64, 64) mean_preprocessor = MeanPreprocessor(means["R"], means["G"], means["B"]) image_to_array_preprocessor = ImageToArrayPreprocessor() # initialize the training and validation dataset generators train_gen = HDF5DatasetGenerator( config.TRAIN_HDF5, 64, augmentation=augmentation, preprocessors=[simple_preprocessor, mean_preprocessor, image_to_array_preprocessor], classes=config.NUM_CLASSES, ) val_gen = HDF5DatasetGenerator( config.VAL_HDF5, 64, preprocessors=[simple_preprocessor, mean_preprocessor, image_to_array_preprocessor], classes=config.NUM_CLASSES, ) # if there is no specific model checkpoint supplied, # then initialize the network and compile the model if args["model"] is None: print("[INFO] compiling model...") model = DeeperGoogLeNet.build(width=64, height=64, depth=3, classes=config.NUM_CLASSES, reg=0.0002) opt = Adam(1e-3) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) # otherwise, load the checkpoint from disk else: print("[INFO] loading {}...".format(args["model"])) model = load_model(args["model"]) # update the learning rate print("[INFO] old learning rate: {}".format(K.get_value(model.optimizer.lr))) K.set_value(model.optimizer.lr, 1e-5) print("[INFO] new learning rate: {}".format(K.get_value(model.optimizer.lr))) # construct the set of callbacks callbacks = [ EpochCheckpoint(args["checkpoints"], every=5, start_at=args["start_epoch"]), TrainingMonitor(config.FIG_PATH, json_path=config.JSON_PATH, start_at=args["start_epoch"]), ] # train the network model.fit_generator( train_gen.generator(), steps_per_epoch=train_gen.num_images // 64, validation_data=val_gen.generator(), validation_steps=val_gen.num_images // 64, epochs=10, max_queue_size=10, callbacks=callbacks, verbose=1, ) # close the databases train_gen.close() val_gen.close()
def test_save_model_with_dynamic_loss_scaling( self, strategy_fn, h5=False, use_v1_loss_scale_optimizer=False): # TODO(reedwm): Support and test saving model with a mixed_[b]float16 policy # as well. strategy = strategy_fn() if (isinstance(strategy, tf.distribute.MirroredStrategy) and not tf.executing_eagerly()): # TODO(b/121381184): Enable running the test in this case. return # Create and run model. with strategy.scope(): x = layers.Input(shape=(2, ), batch_size=2, dtype=tf.float32) y = mp_test_util.MultiplyLayer()(x) model = models.Model(inputs=x, outputs=y) opt = gradient_descent.SGD(1.) if use_v1_loss_scale_optimizer: loss_scale = tf.mixed_precision.experimental.DynamicLossScale( initial_loss_scale=1., increment_period=2.) opt = loss_scale_optimizer.LossScaleOptimizerV1( opt, loss_scale) else: opt = loss_scale_optimizer.LossScaleOptimizer( opt, initial_scale=1., dynamic_growth_steps=2.) model.compile(optimizer=opt, loss='mse', run_eagerly=testing_utils.should_run_eagerly()) # Run for 3 steps (6 examples with a batch size of 2) model.fit(np.ones((6, 2)), np.zeros((6, 2)), batch_size=2) self.assertEqual(backend.get_value(opt.loss_scale), 2) self.assertEqual(backend.get_value(opt.dynamic_counter), 1) (weight, ) = model.trainable_weights orig_weight = backend.get_value(weight) # Save model weights. save_path = os.path.join(self.get_temp_dir(), 'model') model.save(save_path, save_format='h5' if h5 else 'tf') # Run model again for 1 step (2 examples with a batch size of 2) model.fit(np.ones((2, 2)), np.zeros((2, 2)), batch_size=2) new_weight = backend.get_value(weight) self.assertNotEqual(new_weight, orig_weight) self.assertEqual(backend.get_value(opt.loss_scale), 4) self.assertEqual(backend.get_value(opt.dynamic_counter), 0) # Load model weights and ensure loss scale weights are restored. model = save.load_model( save_path, custom_objects={'MultiplyLayer': mp_test_util.MultiplyLayer}) (weight, ) = model.trainable_weights loaded_weight = backend.get_value(weight) self.assertEqual(loaded_weight, orig_weight) # Currently the loss scale isn't always saved when the model is saved with # Model.save(). So we assert the loss scale either has the value when it was # saved, or the value it was initialized with. # TODO(reedwm): Always save/restore the loss scale with Model.save(). self.assertIn(backend.get_value(model.optimizer.loss_scale), (1, 2)) self.assertIn(backend.get_value(model.optimizer.dynamic_counter), (0, 1)) # Test optimizer attributes and type self.assertEqual(model.optimizer.initial_scale, 1.) self.assertEqual(model.optimizer.dynamic_growth_steps, 2.) self.assertEqual(type(model.optimizer), loss_scale_optimizer.LossScaleOptimizer)