def get_config(self):
     config = {'lr': float(K.get_value(self.lr)),
               'momentum': float(K.get_value(self.momentum)),
               'decay': float(K.get_value(self.decay)),
               'nesterov': self.nesterov}
     base_config = super(SimpleSGD, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
Beispiel #2
0
 def get_config(self):
     config = {
         'scale_factor': float(K.get_value(self.scale_factor)),
         'shift_factor': float(K.get_value(self.shift_factor)),
     }
     base_config = super(AddLighting, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
Beispiel #3
0
    def save_weights(self, filepath, overwrite=False):
        # Save weights to HDF5
        import h5py
        import os.path
        # if file exists and should not be overwritten
        if not overwrite and os.path.isfile(filepath):
            import sys
            get_input = input
            if sys.version_info[:2] <= (2, 7):
                get_input = raw_input
            overwrite = get_input('[WARNING] %s already exists - overwrite? [y/n]' % (filepath))
            while overwrite not in ['y', 'n']:
                overwrite = get_input('Enter "y" (overwrite) or "n" (cancel).')
            if overwrite == 'n':
                return
            print('[TIP] Next time specify overwrite=True in save_weights!')

        f = h5py.File(filepath, 'w')
        biases = [K.get_value(x) for x in self.bs]
        weights= [K.get_value(w) for w in self.Ws]
        f.attrs['nb_params'] = len(weights)

        for n, param in enumerate(biases):
            param_name = 'param_bias_{}'.format(n)
            param_dset = f.create_dataset(param_name, param.shape, dtype=param.dtype)
            param_dset[:] = param
        
        for n, param in enumerate(weights):
            param_name = 'param_weight_{}'.format(n)
            param_dset = f.create_dataset(param_name, param.shape, dtype=param.dtype)
            param_dset[:] = param
        f.flush()
        f.close()
Beispiel #4
0
    def on_epoch_end(self, epoch, logs={}):
        logs['lr'] = K.get_value(self.model.optimizer.lr)
        current = logs.get(self.monitor)
        if current is None:
            warnings.warn('Learning Rate Plateau Reducing requires %s available!' %
                          self.monitor, RuntimeWarning)
        else:
            if self.in_cooldown():
                self.cooldown_counter -= 1
                self.wait = 0

            if self.monitor_op(current, self.best):
                self.best = current
                self.wait = 0
            elif not self.in_cooldown():
                if self.wait >= self.patience:
                    old_lr = float(K.get_value(self.model.optimizer.lr))
                    if old_lr > self.min_lr + self.lr_epsilon:
                        new_lr = old_lr * self.factor
                        new_lr = max(new_lr, self.min_lr)
                        K.set_value(self.model.optimizer.lr, new_lr)
                        if self.verbose > 0:
                            print('\nEpoch %05d: reducing learning rate to %s.' % (epoch, new_lr))
                        self.cooldown_counter = self.cooldown
                        self.wait = 0
                self.wait += 1
Beispiel #5
0
def test_ReduceLROnPlateau():
    np.random.seed(1337)
    (X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
                                                         num_test=test_samples,
                                                         input_shape=(input_dim,),
                                                         classification=True,
                                                         num_classes=num_class)
    y_test = np_utils.to_categorical(y_test)
    y_train = np_utils.to_categorical(y_train)

    def make_model():
        np.random.seed(1337)
        model = Sequential()
        model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
        model.add(Dense(num_class, activation='softmax'))

        model.compile(loss='categorical_crossentropy',
                      optimizer=optimizers.SGD(lr=0.1),
                      metrics=['accuracy'])
        return model

    model = make_model()

    # This should reduce the LR after the first epoch (due to high epsilon).
    cbks = [callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, epsilon=10, patience=1, cooldown=5)]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, epochs=5, verbose=2)
    assert np.allclose(float(K.get_value(model.optimizer.lr)), 0.01, atol=K.epsilon())

    model = make_model()
    cbks = [callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, epsilon=0, patience=1, cooldown=5)]
    model.fit(X_train, y_train, batch_size=batch_size,
              validation_data=(X_test, y_test), callbacks=cbks, epochs=5, verbose=2)
    assert np.allclose(float(K.get_value(model.optimizer.lr)), 0.1, atol=K.epsilon())
 def get_config(self):
     config = {'lr': float(K.get_value(self.lr)),
               'momentum': float(K.get_value(self.momentum)),
               'nesterov': self.nesterov,
               'decay_block': float(self.decy_block)}
     base_config = super(SGD_step_decay, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
Beispiel #7
0
	def train(self, model, data):
		""" Fits the given model on a batch of data.
		"""
		kur_optimizer = model.compiled['train']['kur_optimizer']
		if kur_optimizer.scale_rate:
			if kur_optimizer.scale_rate in data:
				import keras.backend as K		# pylint: disable=import-error
				factor = numpy.mean(data[kur_optimizer.scale_rate])
				if kur_optimizer.scale_mode == 'sqrt':
					factor = factor ** 0.5
				keras_optimizer = kur_optimizer.optimizer
				K.set_value(
					keras_optimizer.lr,
					K.get_value(keras_optimizer.lr) * factor
				)
				result = self.run_batch(model, data, 'train', True)
				K.set_value(
					keras_optimizer.lr,
					K.get_value(keras_optimizer.lr) / factor
				)
				return result
			else:
				logger.warning('The optimizer "scale_rate" was specified, but '
					'no such data column was found: %s. Ignoring this.',
					kur_optimizer.scale_rate)
		return self.run_batch(model, data, 'train', True)
Beispiel #8
0
 def get_config(self):
     config = {
         'sigma': self.sigma,
         'threshold': float(K.get_value(self.threshold)),
         'smooth_threshold': float(K.get_value(self.smooth_threshold)),
     }
     base_config = super(Selection, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
Beispiel #9
0
    def _adjust_learning_rate(self, epoch):
        old_lr = K.get_value(self.model.optimizer.lr)
        new_lr = self.initial_lr * self.multiplier(epoch)
        K.set_value(self.model.optimizer.lr, new_lr)

        if hasattr(self.model.optimizer, 'momentum') and self.momentum_correction:
            # See the paper cited above for more information about momentum correction.
            self.restore_momentum = K.get_value(self.model.optimizer.momentum)
            K.set_value(self.model.optimizer.momentum,
                        self.restore_momentum * new_lr / old_lr)
Beispiel #10
0
def test_clone_optimizer():
    lr, momentum, clipnorm, clipvalue = np.random.random(size=4)
    optimizer = SGD(lr=lr, momentum=momentum, clipnorm=clipnorm, clipvalue=clipvalue)
    clone = clone_optimizer(optimizer)

    assert isinstance(clone, SGD)
    assert K.get_value(optimizer.lr) == K.get_value(clone.lr)
    assert K.get_value(optimizer.momentum) == K.get_value(clone.momentum)
    assert optimizer.clipnorm == clone.clipnorm
    assert optimizer.clipvalue == clone.clipvalue
Beispiel #11
0
    def get_config(self):
        config = {
	    "lr": float(K.get_value(self.lr)),
	    "beta_1": float(K.get_value(self.beta_1)),
	    "beta_2": float(K.get_value(self.beta_2)),
            "beta_3": float(K.get_value(self.beta_3)),
	    "epsilon": self.epsilon,
            "decay": float(K.get_value(self.decay))
        }
        base_config = super(Eve, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))
Beispiel #12
0
    def on_epoch_end(self, epoch, logs=None):
        super(LearningRateWarmupCallback, self).on_epoch_end(epoch, logs)

        if epoch == self.end_epoch - 1 and self.verbose > 0:
            new_lr = K.get_value(self.model.optimizer.lr)
            print('\nEpoch %d: finished gradual learning rate warmup to %g.' %
                  (epoch + 1, new_lr))
    def on_batch_begin(self, batch, logs={}):
        open_all_gates()

        rands = np.random.uniform(size=len(add_tables))
        for t, rand in zip(add_tables, rands):
            if rand < K.get_value(t["death_rate"]):
                K.set_value(t["gate"], 0)
Beispiel #14
0
	def build(self, input_shape):
		input_dim = input_shape[-1]
		W1 = weight((input_dim, 4 * self.hidden_dim,), init=self.init, regularizer=self.W_regularizer, name='{}_W1'.format(self.name))
		W2 = weight((self.hidden_dim, self.output_dim), init=self.init, regularizer=self.W_regularizer, name='{}_W2'.format(self.name))
		U = weight((self.hidden_dim, 4 * self.hidden_dim,), init=self.inner_init, regularizer=self.U_regularizer, name='{}_U'.format(self.name))
		b1 = np.concatenate([np.zeros(self.hidden_dim), K.get_value(self.forget_bias_init((self.hidden_dim,))), np.zeros(2 * self.hidden_dim)])
		b1 = weight(b1, regularizer=self.b_regularizer, name='{}_b1'.format(self.name))
		b2 = weight((self.output_dim,), init='zero', regularizer=self.b_regularizer, name='{}_b2'.format(self.name))
		h = (-1, self.hidden_dim)
		c = (-1, self.hidden_dim)

		def step(x, states, weights):
			h_tm1, c_tm1 = states
			W1, W2, U, b1, b2 = weights
			z = K.dot(x, W1) + K.dot(h_tm1, U) + b1
			z0 = z[:, :self.hidden_dim]
			z1 = z[:, self.hidden_dim: 2 * self.hidden_dim]
			z2 = z[:, 2 * self.hidden_dim: 3 * self.hidden_dim]
			z3 = z[:, 3 * self.hidden_dim:]
			i = self.inner_activation(z0)
			f = self.inner_activation(z1)
			c = f * c_tm1 + i * self.activation(z2)
			o = self.inner_activation(z3)
			h = o * self.activation(c)
			y = self.activation(K.dot(h, W2) + b2)
			return y, [h, c]

		self.step = step
		self.states = [h, c]
		self.weights = [W1, W2, U, b1, b2]
		super(LSTMCell, self).build(input_shape)
Beispiel #15
0
 def build(self, input_shape):
     stack_size = input_shape[1]
     self.W_shape = (stack_size, self.nb_filter, self.nb_row, self.nb_col)
     w = self.init(self.W_shape)
     self.W = K.variable(K.get_value(w).reshape(self.W_shape))
     self.b = K.zeros((self.nb_filter,))
     self._trainable_weights = [self.W, self.b]
Beispiel #16
0
 def get_config(self):
     config = {
         'sigma': self.sigma,
         'nb_steps': float(K.get_value(self.nb_steps)),
     }
     base_config = super(HighPass, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
def ExponentialMovingAverage_TrainBegin(model):
    # run when training begins
    # ema_trainable_weights_vals save the latest weights of model with ema
    ema_trainable_weights_vals ={}
    for weight in tqdm(model.trainable_weights):
        ema_trainable_weights_vals[weight.name]=K.get_value(weight)
    return ema_trainable_weights_vals
    def reduce_lr(self, current_nb):
        if self.reduction_function == 'linear':
            new_rate = self.reduce_rate
        elif self.reduction_function == 'exponential':
            new_rate = np.power(self.exp_base,
                                current_nb / self.half_life) * self.reduce_rate
        elif self.reduction_function == 'noam':
            new_rate = np.float32(min(float(current_nb) ** self.exp_base,
                                      float(
                                          current_nb) * self.half_life ** self.warmup_exp))

        else:
            raise NotImplementedError(
                'The decay function %s is not implemented.' % str(
                    self.reduction_function))

        if self.reduction_function == 'noam':
            lr = self.initial_lr
        else:
            lr = K.get_value(self.model.optimizer.lr)
        self.new_lr = np.maximum(np.float32(lr * new_rate), self.min_lr)
        K.set_value(self.model.optimizer.lr, self.new_lr)

        if self.reduce_each_epochs and self.verbose > 0:
            logging.info("LR reduction from {0:0.6f} to {1:0.6f}".format(float(lr),
                                                                         float(self.new_lr)))
Beispiel #19
0
 def get_numvals_list(self, key='omega'):
     """ Returns list of numerical values such as for instance omegas in reproducible order """
     variables = self.vars[key]
     numvals = []
     for p in self.weights:
         numval = K.get_value(tf.reshape(variables[p],(-1,)))
         numvals.append(numval)
     return numvals
Beispiel #20
0
 def get_updates(self, params, constraints, loss):
     grads = self.get_gradients(loss, params)
     for param, grad, c in zip(params, grads, constraints):
         grad_tm1 = K.variable(np.zeros(K.get_value(param).shape))
         step_tm1 = K.variable(
             self.init_step*np.ones(K.get_value(param).shape))
         test = grad * grad_tm1
         diff = T.lt(test, 0)
         steps = step_tm1 * (T.eq(test, 0) +
                             T.gt(test, 0) * self.increase +
                             diff * self.decrease)
         step = T.minimum(self.max_step, T.maximum(self.min_step, steps))
         grad = grad - diff * grad
         self.updates.append((param, c(param - T.sgn(grad) * step)))
         self.updates.append((grad_tm1, grad))
         self.updates.append((step_tm1, step))
     return self.updates
Beispiel #21
0
 def get_state(self):
     state = []
     vs = self.vars
     for key in vs.keys():
         if key=='oopt': continue
         v = vs[key]
         for p in v.values():
             state.append(K.get_value(p)) # FIXME WhyTF does this not work?
     return state
Beispiel #22
0
    def XXXget_training_function(self, x,y):
        # get trainable weights
        trainable_weights = []
        for layer in self.model.layers:
            trainable_weights += collect_trainable_weights(layer)

        # get the grads - more or less
        weights = [K.variable(np.zeros(K.get_value(p).shape)) for p in trainable_weights]
        training_updates = self.optimizer.get_updates(trainable_weights, self.constraints, self.total_loss)
Beispiel #23
0
    def test_load_model_custom_objects(self):
        hvd.init()

        class TestOptimizer(keras.optimizers.RMSprop):
            def __init__(self, **kwargs):
                super(TestOptimizer, self).__init__(**kwargs)

        with self.test_session() as sess:
            K.set_session(sess)

            opt = TestOptimizer(lr=0.0001)
            opt = hvd.DistributedOptimizer(opt)

            model = keras.models.Sequential()
            model.add(keras.layers.Dense(2, input_shape=(3,)))
            model.add(keras.layers.RepeatVector(3))
            model.add(keras.layers.TimeDistributed(keras.layers.Dense(3)))
            model.compile(loss=keras.losses.MSE,
                          optimizer=opt,
                          metrics=[keras.metrics.categorical_accuracy],
                          sample_weight_mode='temporal')

            x = np.random.random((1, 3))
            y = np.random.random((1, 3, 3))
            model.train_on_batch(x, y)

            _, fname = tempfile.mkstemp('.h5')
            model.save(fname)

            custom_objects = {
                'TestOptimizer': lambda **kwargs: hvd.DistributedOptimizer(
                    TestOptimizer(**kwargs))
            }
            new_model = hvd.load_model(fname, custom_objects=custom_objects)
            new_opt = new_model.optimizer
            os.remove(fname)

            self.assertEqual(type(new_opt).__module__, 'horovod.keras')
            self.assertEqual(type(new_opt).__name__, 'TestOptimizer')
            self.assertEqual(K.get_value(opt.lr), K.get_value(new_opt.lr))
            self.assertEqual(len(opt.get_weights()), len(new_opt.get_weights()))
            for weights, new_weights in zip(opt.get_weights(),
                                            new_opt.get_weights()):
                self.assertListEqual(weights.tolist(), new_weights.tolist())
Beispiel #24
0
 def on_batch_begin(self, batch, logs={}):
     # print self.batch_num
     for i in xrange(len(self.batch_point)):
         if self.batch_num < self.batch_point[i]:
             break
         elif self.batch_num == self.batch_point[i]:
             if i < len(self.lr):
                 K.set_value(self.model.optimizer.lr, self.lr[i])
             print 'current lr:', K.get_value(self.model.optimizer.lr)
     self.batch_num += 1
Beispiel #25
0
 def on_epoch_end(self, epoch, logs={}):
     report = {}
     report['epoch'] = epoch
     for k, v in logs.items():
         report[k] = v
     if hasattr(self.model.optimizer, 'lr'):
         report['lr'] = K.get_value(self.model.optimizer.lr).tolist()
     self.task.push('output.training_history', report)
     self.task.set('output.elapsed_time', "%.2fs"%self.elapsed_time)
     self.task.set('output.last_epoch_update_time', datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
Beispiel #26
0
def test_automatic_lr_scheduler():
    optimizer = Adam(lr=0.1)
    lr_scheduler = AutomaticLearningRateScheduler(optimizer, 'loss')
    lr_scheduler.on_train_begin()
    for i in range(3):
        lr_scheduler.on_epoch_begin(i)
        lr_scheduler.on_batch_end(0, {'loss': 1/(i+1)})
        lr_scheduler.on_epoch_end(i)
    o = 3
    assert np.allclose(K.get_value(optimizer.lr), 0.1)

    for i in range(o, o+5):
        print(i)
        lr_scheduler.on_epoch_begin(i)
        lr_scheduler.on_batch_end(0, {'loss': 1})
        lr_scheduler.on_epoch_end(i)
    print(lr_scheduler.current_best)
    print(lr_scheduler.current_best_epoch)
    assert np.allclose(K.get_value(optimizer.lr), 0.1 * lr_scheduler.factor)
Beispiel #27
0
def VGG_16(weights_path=None):
    model = Sequential()
    model.add(ZeroPadding2D((1, 1), input_shape=(3, 224, 224)))
    model.add(Convolution2D(64, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(64, 3, 3, activation="relu"))
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(128, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(128, 3, 3, activation="relu"))
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(256, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(256, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(256, 3, 3, activation="relu"))
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(512, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(512, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(512, 3, 3, activation="relu"))
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(512, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(512, 3, 3, activation="relu"))
    model.add(ZeroPadding2D((1, 1)))
    model.add(Convolution2D(512, 3, 3, activation="relu"))
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    model.add(Flatten())
    model.add(Dense(4096, activation="relu"))
    model.add(Dropout(0.5))
    model.add(Dense(4096, activation="relu"))
    model.add(Dropout(0.5))
    model.add(Dense(1000, activation="softmax"))

    if weights_path:
        model.load_weights(weights_path)
        ops = []
        for layer in model.layers:
            if layer.__class__.__name__ in ["Convolution1D", "Convolution2D"]:
                original_w = K.get_value(layer.W)
                converted_w = convert_kernel(original_w)
                ops.append(tf.assign(layer.W, converted_w).op)
                K.get_session().run(ops)
    return model
def get_states(model):
	all_states = [] 
	for layer in model.layers:
		if hasattr(layer,"states"):
			layer_states = []
			for state in layer.states:
				#print(K.get_value(state)[0][0:3])
				layer_states.append(K.get_value(state))
			all_states.append(layer_states)
	# print(all_states)
	return all_states
Beispiel #29
0
def show_batch_normalization_layer(layer):
    """Serialize batch normalization layer to dict"""
    layer_axis = None
    if isinstance(layer.axis, int):
        layer_axis = layer.axis
    else:
        assert len(layer.axis) == 1
        layer_axis = layer.axis[0]
    assert layer_axis == -1 or layer_axis + 1 == len(layer.input_shape)
    moving_mean = K.get_value(layer.moving_mean)
    moving_variance = K.get_value(layer.moving_variance)
    result = {}
    result['moving_mean'] = encode_floats(moving_mean)
    result['moving_variance'] = encode_floats(moving_variance)
    if layer.center:
        beta = K.get_value(layer.beta)
        result['beta'] = encode_floats(beta)
    if layer.scale:
        gamma = K.get_value(layer.gamma)
        result['gamma'] = encode_floats(gamma)
    return result
Beispiel #30
0
def _runner(init, shape, target_mean=None, target_std=None,
            target_max=None, target_min=None):
    variable = init(shape)
    output = K.get_value(variable)
    lim = 1e-2
    if target_std is not None:
        assert abs(output.std() - target_std) < lim
    if target_mean is not None:
        assert abs(output.mean() - target_mean) < lim
    if target_max is not None:
        assert abs(output.max() - target_max) < lim
    if target_min is not None:
        assert abs(output.min() - target_min) < lim
Beispiel #31
0
    def on_batch_end(self, batch, logs=None):
        if self.current_epoch_ > 1:
            return

        if self.use_validation_set:
            X, Y = self.validation_data[0], self.validation_data[1]

            # use 5 random batches from test set for fast approximate of loss
            num_samples = self.batch_size * self.validation_sample_rate

            if num_samples > X.shape[0]:
                num_samples = X.shape[0]

            idx = np.random.choice(X.shape[0], num_samples, replace=False)
            x = X[idx]
            y = Y[idx]

            values = self.model.evaluate(x, y, batch_size=self.batch_size, verbose=False)
            loss = values[0]
        else:
            loss = logs['loss']

        # smooth the loss value and bias correct
        running_loss = self.loss_smoothing_beta * loss + (
                1. - self.loss_smoothing_beta) * loss
        running_loss = running_loss / (
                1. - self.loss_smoothing_beta ** self.current_batch_)

        # stop logging if loss is too large
        if self.current_batch_ > 1 and self.stopping_criterion_factor is not None and (
                running_loss >
                self.stopping_criterion_factor * self.best_loss_):

            if self.verbose:
                print(" - LRFinder: Skipping iteration since loss is %d times as large as best loss (%0.4f)"
                      % (self.stopping_criterion_factor, self.best_loss_))
            return

        if running_loss < self.best_loss_ or self.current_batch_ == 1:
            self.best_loss_ = running_loss

        current_lr = K.get_value(self.model.optimizer.lr)

        self.history.setdefault('running_loss_', []).append(running_loss)
        if self.lr_scale == 'exp':
            self.history.setdefault('log_lrs', []).append(np.log10(current_lr))
        else:
            self.history.setdefault('log_lrs', []).append(current_lr)

        # compute the lr for the next batch and update the optimizer lr
        if self.lr_scale == 'exp':
            current_lr *= self.lr_multiplier_
        else:
            current_lr = self.lr_multiplier_[self.current_batch_ - 1]

        K.set_value(self.model.optimizer.lr, current_lr)

        # save the other metrics as well
        for k, v in logs.items():
            self.history.setdefault(k, []).append(v)

        if self.verbose:
            if self.use_validation_set:
                print(" - LRFinder: val_loss: %1.4f - lr = %1.8f " %
                      (values[0], current_lr))
            else:
                print(" - LRFinder: lr = %1.8f " % current_lr)
Beispiel #32
0
 def on_epoch_begin(self, epoch, logs={}):
     lr = float(K.get_value(self.model.optimizer.lr))
     print('VAE lr:', lr)
     print('VAE adam:', self.model.optimizer.get_config())
Beispiel #33
0
def train(BATCH_SIZE, ENC_WEIGHTS, DEC_WEIGHTS):
    print ("Loading data definitions...")
    frames_source = hkl.load(os.path.join(DATA_DIR, 'sources_train_208.hkl'))
    videos_list = get_video_lists(frames_source=frames_source, stride=4)
    n_videos = videos_list.shape[0]

    # Setup test
    val_frames_source = hkl.load(os.path.join(VAL_DATA_DIR, 'sources_val_208.hkl'))
    val_videos_list = get_video_lists(frames_source=val_frames_source, stride=(int(VIDEO_LENGTH/2)))
    n_val_videos = val_videos_list.shape[0]

    if RAM_DECIMATE:
        frames = load_to_RAM(frames_source=frames_source)

    if SHUFFLE:
        # Shuffle images to aid generalization
        videos_list = np.random.permutation(videos_list)

    # Build the Spatio-temporal Autoencoder
    print ("Creating models...")
    encoder = encoder_model()
    print (encoder.summary())

    decoder = decoder_model()
    autoencoder = autoencoder_model(encoder, decoder)
    autoencoder.compile(loss="mean_squared_error", optimizer=OPTIM_A)

    run_utilities(encoder, decoder, autoencoder, ENC_WEIGHTS, DEC_WEIGHTS)

    NB_ITERATIONS = int(n_videos/BATCH_SIZE)
    # NB_ITERATIONS = 5
    NB_VAL_ITERATIONS = int(n_val_videos / BATCH_SIZE)

    # Setup TensorBoard Callback
    TC = tb_callback.TensorBoard(log_dir=TF_LOG_DIR, histogram_freq=0, write_graph=False, write_images=False)
    LRS = lrs_callback.LearningRateScheduler(schedule=schedule)
    LRS.set_model(autoencoder)

    print ("Beginning Training...")
    # Begin Training
    for epoch in range(1, NB_EPOCHS_AUTOENCODER):
        if epoch == 21:
            autoencoder.compile(loss="mean_absolute_error", optimizer=OPTIM_B)

        print("\n\nEpoch ", epoch)
        loss = []
        val_loss = []

        # Set learning rate every epoch
        LRS.on_epoch_begin(epoch=epoch)
        lr = K.get_value(autoencoder.optimizer.lr)
        print ("Learning rate: " + str(lr))

        for index in range(NB_ITERATIONS):
            # Train Autoencoder
            if RAM_DECIMATE:
                X = load_X_RAM(videos_list, index, frames)
            else:
                X = load_X(videos_list, index, DATA_DIR, IMG_SIZE)
            X_train = np.flip(X[:, 0 : int(VIDEO_LENGTH/2)], axis=1)
            y_train = X[:, int(VIDEO_LENGTH/2) :]
            loss.append(autoencoder.train_on_batch(X_train, y_train))

            arrow = int(index / (NB_ITERATIONS / 40))
            stdout.write("\rIter: " + str(index) + "/" + str(NB_ITERATIONS-1) + "  " +
                         "loss: " + str(loss[len(loss)-1]) +
                         "\t    [" + "{0}>".format("="*(arrow)))
            stdout.flush()

        if SAVE_GENERATED_IMAGES:
            # Save generated images to file
            predicted_images = autoencoder.predict(X_train, verbose=0)
            voila = np.concatenate((X_train, y_train), axis=1)
            truth_seq = arrange_images(voila)
            pred_seq = arrange_images(np.concatenate((X_train, predicted_images), axis=1))

            truth_seq = truth_seq * 127.5 + 127.5
            pred_seq = pred_seq * 127.5 + 127.5

            if epoch == 1:
                cv2.imwrite(os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + "_truth.png"), truth_seq)
            cv2.imwrite(os.path.join(GEN_IMAGES_DIR, str(epoch) + "_" + str(index) + "_pred.png"), pred_seq)

        # Run over test data
        print ('')
        for index in range(NB_VAL_ITERATIONS):
            X = load_X(val_videos_list, index, VAL_DATA_DIR, IMG_SIZE)
            X_val = np.flip(X[:, 0: int(VIDEO_LENGTH / 2)], axis=1)
            y_val = X[:, int(VIDEO_LENGTH / 2):]
            val_loss.append(autoencoder.test_on_batch(X_val, y_val))

            arrow = int(index / (NB_VAL_ITERATIONS / 40))
            stdout.write("\rIter: " + str(index) + "/" + str(NB_VAL_ITERATIONS - 1) + "  " +
                         "test_loss: " + str(val_loss[len(val_loss) - 1]) +
                         "\t    [" + "{0}>".format("=" * (arrow)))
            stdout.flush()

        # then after each epoch/iteration
        avg_loss = sum(loss)/len(loss)
        avg_val_loss = sum(val_loss) / len(val_loss)
        logs = {'loss': avg_loss, 'val_loss': avg_val_loss}
        TC.on_epoch_end(epoch, logs)

        # Log the losses
        with open(os.path.join(LOG_DIR, 'losses_gen.json'), 'a') as log_file:
            log_file.write("{\"epoch\":%d, \"train_loss\":%f, \"val_loss\":%f}\n" % (epoch, avg_loss, avg_val_loss))

            print("\nAvg train loss: " + str(avg_loss) + " Avg val loss: " + str(avg_val_loss))

        # Save model weights per epoch to file
        if epoch>15 and epoch<21:
            encoder.save_weights(os.path.join(CHECKPOINT_DIR, 'encoder_epoch_' + str(epoch) + '.h5'), True)
            decoder.save_weights(os.path.join(CHECKPOINT_DIR, 'decoder_epoch_' + str(epoch) + '.h5'), True)
        if epoch>25:
            encoder.save_weights(os.path.join(CHECKPOINT_DIR, 'encoder_epoch_' + str(epoch) + '.h5'), True)
            decoder.save_weights(os.path.join(CHECKPOINT_DIR, 'decoder_epoch_' + str(epoch) + '.h5'), True)
Beispiel #34
0
def get_tensor_value(x):
    if isinstance(x, tf.Tensor):
        return K.get_value(x)
    return x
Beispiel #35
0
    float_data[i, :] = values

# Train Set
float_data = shuffel(float_data)

#Nrmalisation train_data
float_data[:, :-1] = dp.zero_mean_normalization(float_data[:, :-1])
data = float_data[:, :-1]

train_data = data[150:]
label_data = float_data[150:, -1]

#validation data
# MCAR
train_data_voll = K.dropout(train_data, 0.0001, None, None)
train_data_voll = K.get_value(train_data_voll)

train_data_20_MCAR = K.dropout(train_data, 0.2, None, None)
train_data_20_MCAR = K.get_value(train_data_20_MCAR)
train_data_20_MCAR *= 0.8

train_data_40_MCAR = K.dropout(train_data, 0.4, None, None)
train_data_40_MCAR = K.get_value(train_data_40_MCAR)
train_data_40_MCAR *= 0.6

#MAR
sort_float_data = sorted(float_data[150:], key=lambda x: x[1])
sort_float_data = np.array(sort_float_data)
sort_train_data = sort_float_data[:, :-1]
label_data_MAR = sort_float_data[:, -1]
Beispiel #36
0
def main(args):
    # set the image parameters
    img_rows = args.img_rows
    img_cols = args.img_cols
    img_dim_color = args.img_channels
    # mix_prop = 1.0                                                    # set the value of the mixing proportion

    #############################################################################################################
    ##################################  DEFINING MODEL  ##########################################################
    ##############################################################################################################
    model_alex = cnn_hybrid_color_single(img_rows, img_cols,
                                         img_dim_color)  # load the model

    # model_final = Model(model_alex.input, model_alex.output)  # specify the input and output of the model
    model_final = model_alex
    print(model_final.summary())  # print the model summary

    plot_model(model_final,
               to_file='./NIN_hybrid_bin_resnet_1x1-class',
               show_shapes=True)  # save the model summary as a png file

    lr = args.learning_rate  # set the learning rate

    # set the optimizer
    optimizer = SGD(lr=lr, decay=1e-6, momentum=0.9)

    # model compilation
    model_final.compile(optimizer=optimizer,
                        loss='sparse_categorical_crossentropy',
                        metrics=['accuracy'])

    # print the value of the learning rate
    print(K.get_value(optimizer.lr))

    # --------------------------------------------------
    #############################################################################################################
    ########################## GETTING TRAINING DATA AND TESTING DATA  ##########################################
    ##############################################################################################################

    # get the training data by calling the pairs function
    # read the training data

    train_pairs_r, training_data_r, training_label_r = read_pairs(
        args.tr_img_lab_r)
    train_pairs_l, training_data_l, training_label_l = read_pairs(
        args.tr_img_lab_l)

    assert len(training_data_r) == len(training_data_l)

    # combine the left and right image in the training data to make a X x Y x 6 tensor
    training_data = []
    for i in range(len(training_data_r)):
        # define the stereo pair
        stereo_pair = [training_data_r[i], training_data_l[i]]
        training_data.append(stereo_pair)

    batch_num = 0

    # initialize the live samples and fake samples
    live_samples_ub = 0
    attack_samples_ub = 0

    live_samples = []
    live_labels = []
    attack_samples = []
    attack_labels = []

    # separate the live samples and fake samples to balance the both classes, i.e. live class and fake class
    assert len(training_label_r) == len(training_label_l)

    for i in range(len(training_data)):
        if training_label_r[i] == 0:
            live_samples.append(training_data[i])
            live_labels.append(training_label_r[i])

            live_samples_ub += 1
        elif (training_label_r[i] == 1) | (training_label_r[i] == 2) | (
                training_label_r[i] == 4):  # protocol_2
            attack_samples.append(training_data[i])
            attack_labels.append(training_label_r[i])

            attack_samples_ub += 1

    print("Live samples are %g ,\t attack samples are %g" %
          (live_samples_ub, attack_samples_ub))

    # compute the difference; the live samples are always less than the fake samples in our case
    diff = 0
    if live_samples_ub < attack_samples_ub:
        # compute the ratio
        diff = np.int(attack_samples_ub / live_samples_ub)
        print("The difference is :%g " % (diff))
    else:
        ValueError("The fake samples are less than then live samples")

    # number of times the dataset has to be copied:
    live_samples_b = live_samples
    live_labels_b = live_labels
    for i in range(diff - 1):
        # print("length before balancing: %g" %len(live_samples_b))
        sl_copy = live_samples.copy()
        ll_copy = live_labels.copy()

        live_samples_b = live_samples_b + sl_copy
        live_labels_b = live_labels_b + ll_copy
        # print("length after balancing: %g" % len(live_samples_b))

    # balanced data
    training_data_balanced = live_samples_b + attack_samples
    training_label_balanced = live_labels_b + attack_labels

    print("Balanced data samples: %g" % len(training_data_balanced))

    # get the length of the training data
    len_tr = len(training_data_balanced)

    # get the number equal to the length of the training data
    indices_tr = np.arange(len_tr)
    np.random.shuffle(indices_tr)

    # initialize the image counter
    images_read = 0
    train_img_data_r = []
    train_img_data_l = []

    for i in indices_tr:
        if training_label_balanced[i] > 0:
            training_label_balanced[i] = 1

        train_img_data_r.append(
            [training_data_balanced[i][0],
             training_label_balanced[i]])  # read the right image
        train_img_data_l.append(
            [training_data_balanced[i][1],
             training_label_balanced[i]])  # read the left image

        # print(training_data_balanced[i][1])
        # cv2.imshow('img1', cv2.imread(training_data_balanced[i][0]))
        # cv2.waitKey()
        # cv2.imshow('img2', cv2.imread(training_data_balanced[i][1]))
        # cv2.waitKey()

        images_read += 1
        sys.stdout.write('train images read = {0}\r'.format(images_read))
        sys.stdout.flush()

    ############################################################################################################

    # read the test data
    test_pairs, test_data_r, test_labels_r = read_pairs(args.tst_img_lab_r)
    test_pairs, test_data_l, test_labels_l = read_pairs(args.tst_img_lab_l)

    assert len(test_data_r) == len(test_data_l)

    # combine the left and right image in the training data to make a X x Y x 6 tensor
    test_data = []
    for i in range(len(test_data_r)):
        # define the stereo pair
        stereo_pair_t = [test_data_r[i], test_data_l[i]]
        test_data.append(stereo_pair_t)

    test_labels = test_labels_r

    images_read = 0

    # get the length of the training data
    len_test = len(test_data)

    # get the number equal to the length of the training data
    indices_test = np.arange(len_test)

    test_img_data_r = []
    test_img_data_l = []

    for i in indices_test:

        if test_labels[i] > 0:
            test_labels[i] = 1

        test_img_data_r.append([test_data[i][0],
                                test_labels[i]])  # read the right test image
        test_img_data_l.append([test_data[i][1],
                                test_labels[i]])  # red the left test image
        images_read += 1
        sys.stdout.write('test images read = {0}\r'.format(images_read))
        sys.stdout.flush()

    #####################################################################################################
    # make all the data in panda data frame format
    train_df_r = pd.DataFrame(train_img_data_r)
    train_df_r.columns = ['id_r', 'label']

    train_df_l = pd.DataFrame(train_img_data_l)
    train_df_l.columns = ['id_l', 'label']

    test_df_r = pd.DataFrame(test_img_data_r)
    test_df_r.columns = ['id_r', 'label']

    test_df_l = pd.DataFrame(test_img_data_l)
    test_df_l.columns = ['id_l', 'label']

    ########################################################################################################333

    datagen = image.ImageDataGenerator()

    train_generator_r = datagen.flow_from_dataframe(
        dataframe=train_df_r,
        directory=None,
        x_col='id_r',
        y_col='label',
        has_ext=True,
        batch_size=args.batch_size,
        seed=42,
        shuffle=True,
        class_mode="sparse",
        target_size=(args.img_rows, args.img_cols),
        color_mode='grayscale',
        interpolation='nearest',
        drop_duplicates=False)

    train_generator_l = datagen.flow_from_dataframe(
        dataframe=train_df_l,
        directory=None,
        x_col='id_l',
        y_col='label',
        has_ext=True,
        batch_size=args.batch_size,
        seed=42,
        shuffle=True,
        class_mode="sparse",
        target_size=(args.img_rows, args.img_cols),
        color_mode='grayscale',
        interpolation='nearest',
        drop_duplicates=False)

    test_datagen = image.ImageDataGenerator()

    test_generator_r = test_datagen.flow_from_dataframe(
        dataframe=test_df_r,
        directory=None,
        x_col='id_r',
        y_col='label',
        has_ext=True,
        batch_size=args.batch_size,
        seed=42,
        shuffle=False,
        class_mode="sparse",
        target_size=(args.img_rows, args.img_cols),
        color_mode='grayscale',
        interpolation='nearest')

    test_generator_l = test_datagen.flow_from_dataframe(
        dataframe=test_df_l,
        directory=None,
        x_col='id_l',
        y_col='label',
        has_ext=True,
        batch_size=args.batch_size,
        seed=42,
        shuffle=False,
        class_mode="sparse",
        target_size=(args.img_rows, args.img_cols),
        color_mode='grayscale',
        interpolation='nearest')
    #############################################################################################################
    batch_num = 0
    while batch_num < args.max_epochs:

        start_time = time.time()  # initialize the clock
        acc = []
        loss = []
        sub_count = 0

        total_batch = train_generator_r.n // train_generator_r.batch_size

        for i in range(train_generator_r.n // train_generator_r.batch_size):
            x1, y1 = next(train_generator_r)
            x2, y2 = next(train_generator_l)

            # only for DP-3D for comparison
            # disparity_final = []
            #
            # for j in range(x1.shape[0]):
            #     img1 = np.asarray(x1[j])
            #     # img1 = cv2.resize(img1, (img_rows, img_cols),
            #     #                                 interpolation=cv2.INTER_AREA)
            #
            #     img2 = np.asarray(x2[j])
            #     # img2 = cv2.resize(img2, (img_rows, img_cols),
            #     #                                 interpolation=cv2.INTER_AREA)
            #     #
            #     disparity = cv2.subtract(img1, img2)
            #
            #     der_k = np.asarray([[1.0, 2.0, 1.0],
            #                         [0.0, 0.0, 0.0],
            #                         [-1.0, -2.0, -1.0]])
            #
            #     der = cv2.filter2D(img1, -1, kernel=der_k)
            #
            #     disparity_f = disparity / (der + 0.005)
            #
            #     disparity_final.append(disparity_f)
            #
            # # disparity_final = np.asarray(disparity_final).astype('float32')
            # disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'),axis=-1)

            x1 = x1.astype('float32') / 255
            x2 = x2.astype('float32') / 255

            y = y1

            tr_acc1 = model_final.fit([x1, x2], y, epochs=1, verbose=0)
            acc.append(tr_acc1.history['acc'][0])
            loss.append(tr_acc1.history['loss'][0])

            sub_count += 1
            sys.stdout.write('batch_count = {0} of {1} \r'.format(
                sub_count, total_batch))
            sys.stdout.flush()

        train_acc = np.sum(np.asarray(acc)) * 100 / (
            train_generator_r.n // train_generator_r.batch_size)
        train_loss = np.sum(np.asarray(loss)) * 100 / (
            train_generator_r.n // train_generator_r.batch_size)

        print('training_acc: {0} \t training_loss: {1}'.format(
            train_acc, train_loss))

        print(
            '______________________________________________________________________'
        )
        print('Running the evaluations')

        test_acc = []
        test_loss = []
        sub_count = 0

        for i in range(test_generator_r.n // test_generator_r.batch_size):
            x1, y1 = next(test_generator_r)
            x2, y2 = next(test_generator_l)

            # only for DP-3D for comparison
            # disparity_final = []
            #
            # for j in range(x1.shape[0]):
            #     img1 = np.asarray(x1[j])
            #     # img1 = cv2.resize(img1, (img_rows, img_cols),
            #     #                                 interpolation=cv2.INTER_AREA)
            #
            #     img2 = np.asarray(x2[j])
            #     # img2 = cv2.resize(img2, (img_rows, img_cols),
            #     #                                 interpolation=cv2.INTER_AREA)
            #     #
            #     disparity = cv2.subtract(img1, img2)
            #
            #     der_k = np.asarray([[1.0, 2.0, 1.0],
            #                         [0.0, 0.0, 0.0],
            #                         [-1.0, -2.0, -1.0]])
            #
            #     der = cv2.filter2D(img1, -1, kernel=der_k)
            #
            #     disparity_f = disparity / (der + 0.005)
            #
            #     disparity_final.append(disparity_f)
            #
            # # disparity_final = np.asarray(disparity_final).astype('float32')
            # disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'), axis=-1)

            x1 = x1.astype('float32') / 255
            x2 = x2.astype('float32') / 255

            y = y1

            tst_loss, tst_acc1 = model_final.evaluate([x1, x2], y, verbose=0)
            test_acc.append(tst_acc1)
            test_loss.append(tst_loss)
            sub_count += 1
            sys.stdout.write('epoch_count = {0}\r'.format(sub_count))
            sys.stdout.flush()

        test_acc = np.sum(np.asarray(test_acc)) * 100 / (
            test_generator_r.n // test_generator_r.batch_size)
        test_loss = np.sum(np.asarray(test_loss)) * 100 / (
            test_generator_r.n // test_generator_r.batch_size)

        print('test_acc: {0} \t test_loss: {1}'.format(test_acc, test_loss))

        batch_num += 1

        # **********************************************************************************************
        # learning rate schedule update: if learning is done using a single learning give the batch_num below a
        # high value
        if (batch_num == 3) | (batch_num == 5) | (batch_num == 7):
            lr = 0.1 * lr
            K.set_value(optimizer.lr, lr)
            print(K.get_value(optimizer.lr))

        # ************************************************************************************************
        # -----------------------------------------------------------------------------------------------

        end_time = time.time() - start_time

        print("Total time taken %f :" % end_time)

        model_final.save_weights(
            '/home/yaurehman2/Documents/stereo_face_liveness/stereo_ckpt/Conventional/'
            + 'dual_grayscale_input_revised_protocol_2_' +
            str(args.max_epochs) + '.h5')
Beispiel #37
0
    def on_epoch_end(self, epoch, logs={}):
        self.seen += logs.get('size', 0)
        #print([l.name for l in self.model.layers])
        
        if epoch % self.display == 0:
            metrics_log = ''
            for k in self.params['metrics']:
                if k in logs:
                    val = logs[k]
                    if abs(val) > 1e-3:
                        metrics_log += ' - %s: %.4f' % (k, val)
                    else:
                        metrics_log += ' - %s: %.4e' % (k, val)
            #weight = self.model.loss.keywords['weight']
            
            if(self.is_VAE):
                weight = K.get_value(self.model.loss_weights['decoder_for_kl'])
            #print(self.model.get_layer('sample_z').ouput.values())

            inputTensor=[self.model.get_layer('x_true').input]
            #inputTensor=[self.model.get_layer('enc_x_true').input,self.model.get_layer('enc_cond').input]
            
            n_cond_pre=0
            n_embs_input=0
            for l in self.model.layers:
                if('cond_pre' in l.name ):
                    inputTensor.append(self.model.get_layer(l.name).input)
                    n_cond_pre=n_cond_pre+1
            
            for l in self.model.layers:
                if('emb_input' in l.name ):
                    inputTensor.append(self.model.get_layer(l.name).input)
                    n_embs_input=n_embs_input+1
                    print(l.name )

            x_inputs =self.x_train_data
            inputsY=x_inputs[0]
            
            if(n_cond_pre>=1):
                if(n_cond_pre==1):
                    cond_pre=x_inputs[1]
                else:
                    cond_pre=x_inputs[1:(1+n_cond_pre)]
                
                if(n_embs_input>=1):
                    emb_inputs=x_inputs[(1+n_cond_pre):]
                    embModel=self.model.get_layer('embedding_enc')
                    emb_ouputs = embModel.predict(emb_inputs)
                
                    cond = np.concatenate((cond_pre, emb_ouputs), axis=1)
                    input_encoder = [inputsY,cond]
                else:
                    input_encoder = [inputsY,cond_pre]
            elif(n_embs_input>=1):
                emb_inputs=x_inputs[1:]
                embModel=self.model.get_layer('embedding_enc')
                emb_ouputs = embModel.predict(emb_inputs)
                #emb_ouputs=np.squeeze(emb_ouputs, axis=0)
                input_encoder = [inputsY,emb_ouputs]
            else:
                input_encoder=[inputsY]
            
            self.response_model=self.model.get_layer('encoder')
            
            responses=self.response_model.predict(input_encoder)
            if(self.is_VAE):
                responses=responses[0]
           
            #responses=self.model.encoder.predict(self.x_train_data)
            print(np.sum(np.abs(responses),axis=0))
            predictFeaturesInLatentSPace(self.x_conso,self.calendar_info,responses,k=5)
            
            valLoss=logs.get('val_loss')
            
            if(self.is_VAE):
                print('{} Epochs ... {} val_loss {} ... lambda Loss {}'.format(epoch, metrics_log,valLoss,weight))
            else:
                print('{} Epochs ... {}'.format(epoch, metrics_log))
model2 = Model(inputs=new_input, outputs=[new_out_softmax])

model2.summary()

input("Press Enter to continue.")
# Training:
try:
    model2.fit([imgs_train, bots_train], lab_train_ohe,
              batch_size=50, epochs=110, verbose=1,
              validation_data=([imgs_val, bots_val], lab_val_ohe),
              callbacks=[RegScheduler(beta=beta, epoch_count=epoch_count)])

    score, accuracy = model2.evaluate([imgs_test, bots_test], lab_test_ohe, batch_size=100, verbose=0)
    print('Test score:', score)
    print('Test accuracy:', accuracy)
    epc_count= int(K.get_value(epoch_count))+1
    file_name = 'model_exp_'+str(epc_count)+'.h5'
    model_path_save = os.path.join(model_dir, file_name)
    model.save(model_path_save)

    # Layers sizes
    input("Press Enter to continue.")
    bot_lay_size = bots_train.shape[1]
    n_train_imgs = imgs_train.shape[0]
    n_test_imgs = imgs_test.shape[0]
    n_classes = lab_train_ohe.shape[1]
    print("bot lay size:", bot_lay_size)
    print("train_imgs", n_train_imgs)
    print('test imgs', n_test_imgs)
    print("classes", n_classes)
 def on_epoch_end(self, epoch, logs={}):
     max_epoch= 90
     power = 4
     stop = 0
     K.set_value(self.beta, ((1-(epoch/max_epoch)) ** power ) * (1-stop) + stop )
     print('---current beta: %.3f' % K.get_value(beta))
Beispiel #40
0
def scheduler(epoch):
    if epoch % 20 == 0 and epoch != 0:
        lr = K.get_value(model.optimizer.lr)
        K.set_value(model.optimizer.lr, lr * 0.1)
        print("lr changed to {}".format(lr * 0.1))
    return K.get_value(model.optimizer.lr)
Beispiel #41
0
                         3,
                         2, (3, 4, 6), (64, 128, 256, 512),
                         reg=0.0005,
                         dataset="tiny_imagenet")
    opt = SGD(lr=1e-1, momentum=0.9)
    model.compile(loss="binary_crossentropy",
                  optimizer=opt,
                  metrics=["accuracy"])

# otherwise, load the checkpoint from disk
else:
    print("[INFO] loading {}...".format(args["model"]))
    model = load_model(args["model"])

    # update the learning rate
    print("[INFO] old learning rate: {}".format(K.get_value(
        model.optimizer.lr)))
    K.set_value(model.optimizer.lr, 1e-3)
    print("[INFO] new learning rate: {}".format(K.get_value(
        model.optimizer.lr)))

# construct the set of callbacks
figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())])
jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())])
fname = os.path.sep.join(
    [args["weights"], "weights-{epoch:03d}-{val_acc:.4f}.hdf5"])
checkpoint = ModelCheckpoint(fname,
                             monitor="val_acc",
                             mode="max",
                             save_best_only=True,
                             verbose=1)
callbacks = [
Beispiel #42
0
    def test_dynamic_loss_scaling(self,
                                  strategy_fn,
                                  pass_loss_scale_to_policy=False,
                                  get_config=False,
                                  use_v1_loss_scale_optimizer=False):
        strategy = strategy_fn()
        initial_loss_scale = 2.
        batch_size = 4
        expected_gradient = backend.variable([initial_loss_scale / batch_size],
                                             dtype=tf.float16)
        # If this variable is set to True, the model below will have NaN gradients
        have_nan_gradients = backend.variable(False, dtype=tf.bool)
        with strategy.scope():
            opt = gradient_descent.SGD(1.)
            if pass_loss_scale_to_policy:
                loss_scale = tf.mixed_precision.experimental.DynamicLossScale(
                    initial_loss_scale=initial_loss_scale, increment_period=2)
                p = policy.PolicyV1('mixed_float16', loss_scale=loss_scale)
            elif use_v1_loss_scale_optimizer:
                loss_scale = tf.mixed_precision.experimental.DynamicLossScale(
                    initial_loss_scale=initial_loss_scale, increment_period=2)
                p = policy.Policy('mixed_float16')
                opt = loss_scale_optimizer.LossScaleOptimizerV1(
                    opt, loss_scale)
            else:
                p = policy.Policy('mixed_float16')
                opt = loss_scale_optimizer.LossScaleOptimizer(
                    opt,
                    initial_scale=initial_loss_scale,
                    dynamic_growth_steps=2)
            with policy.policy_scope(p):
                x = layers.Input(shape=(1, ),
                                 batch_size=batch_size,
                                 dtype=tf.float16)
                layer = mp_test_util.MultiplyLayer(assert_type=tf.float16)
                y = layer(x)
                identity_with_nan_grads = (
                    mp_test_util.create_identity_with_nan_gradients_fn(
                        have_nan_gradients))
                y = core.Lambda(identity_with_nan_grads)(y)
                identity_with_grad_check_fn = (
                    mp_test_util.create_identity_with_grad_check_fn(
                        expected_dtype=tf.float16,
                        expected_gradient=expected_gradient))
                y = core.Lambda(identity_with_grad_check_fn)(y)
                model = models.Model(inputs=x, outputs=y)
                if get_config:
                    config = model.get_config()
                    model = model.__class__.from_config(
                        config,
                        custom_objects={
                            'MultiplyLayer': mp_test_util.MultiplyLayer
                        })
                    (layer, ) = (
                        layer for layer in model.layers
                        if isinstance(layer, mp_test_util.MultiplyLayer))

                def loss_fn(y_true, y_pred):
                    del y_true
                    return tf.reduce_mean(y_pred)

                model.compile(opt,
                              loss=loss_fn,
                              run_eagerly=testing_utils.should_run_eagerly())

        self.assertEqual(backend.eval(layer.v), 1)
        x = np.ones((batch_size, 1))
        y = np.ones((batch_size, 1))
        dataset = tf.data.Dataset.from_tensor_slices((x, y)).batch(batch_size)
        model.fit(dataset)
        # The variables starts with 1 and has a gradient of 1, so will go down by 1
        # each step.
        self.assertEqual(backend.eval(layer.v), 0)

        model.fit(dataset)
        self.assertEqual(backend.eval(layer.v), -1)

        # There have been two steps without NaNs, so the loss scale will double
        backend.set_value(expected_gradient,
                          backend.get_value(expected_gradient * 2))
        model.fit(dataset)
        self.assertEqual(backend.eval(layer.v), -2)

        # Next test with NaN gradients.
        backend.set_value(have_nan_gradients, True)
        model.fit(dataset)
        # Variable should not be updated
        self.assertEqual(backend.eval(layer.v), -2)

        # Test with finite gradients again
        backend.set_value(have_nan_gradients, False)
        # The loss scale will be halved due to the NaNs, so the gradient will also
        # be halved
        backend.set_value(expected_gradient,
                          backend.get_value(expected_gradient / 2))
        model.fit(dataset)
        self.assertEqual(backend.eval(layer.v), -3)
Beispiel #43
0
 def on_epoch_end(self, epochs, logs={}):
     lr = K.get_value(self.model.optimizer.lr)
     print('\nLearningRate:{:.6f}'.format(lr))
    model = keras.models.load_model(path_to_model,
                                    custom_objects={'psnr': psnr})

    # Check its architecture
    model.summary()

    # Do the prediction
    pred = model.predict(test_x)

    # Print PREDICTION
    print("--PREDICTION--")
    print(pred)

    # Calculate PSNR between ORIGINAL and PREDICTION
    print("--PSNR--")
    psnr_array = K.get_value(psnr(test_y, pred))
    print(psnr_array)

    fig = plt.figure(figsize=(50, 50), facecolor="w")

    plt.subplot(1, 3, 1)
    plt.title("Original", fontsize=80)
    plt.tight_layout()
    plt.imshow(test_y[N_show, :, :])

    plt.subplot(1, 3, 2)
    plt.title("Low resolution", fontsize=80)
    plt.tight_layout()
    plt.imshow(test_x[N_show, :, :])

    plt.subplot(1, 3, 3)
Beispiel #45
0
    def learn(self,
              model,
              model_file=None,
              start_from_scratch=False,
              reference_states=None,
              input_shape=None):
        q_progress = {'q_values': [], 'epochs': []}
        tensorboard = TensorBoard(log_dir="logs/{}".format(time()))

        if model_file is not None and not start_from_scratch:
            model.load_weights(model_file)

        # TODO: find alternative way for calculating reference-q-values
        if reference_states is not None:
            q_progress['q_values'].append(
                self.get_avg_max_q(model,
                                   reference_states=reference_states,
                                   shape=input_shape))
            q_progress['epochs'].append(0)

        # Train
        for e in range(self.q_learning_epochs):

            all_inputs = None
            all_targets = None
            all_q_deltas = []

            for el in self.experience_list:
                # the Q-learning magic happens here...
                inputs, targets, q_delta = el.get_batch(
                    model, single_actions=self.single_actions, p=0.25)

                if input_shape is not None:
                    inputs.reshape((inputs.shape[0], ) + input_shape)

                if (all_inputs is None):
                    all_inputs = inputs
                    all_targets = targets
                else:
                    all_inputs = np.concatenate((all_inputs, inputs), axis=0)
                    all_targets = np.concatenate((all_targets, targets),
                                                 axis=0)

                all_q_deltas += q_delta

            logging.debug("q_deltas: N: %d, min %f, max %f, avg %f" %
                          (len(all_q_deltas), min(all_q_deltas),
                           max(all_q_deltas), statistics.mean(all_q_deltas)))
            # adapt model
            X_train, X_test, y_train, y_test = train_test_split(
                all_inputs, all_targets, test_size=0.2)  #Xrandom_state=42)
            _lr = K.get_value(model.optimizer.lr)
            logging.debug(" Current learning rate (before fit):" + str(_lr))
            #K.set_value(model.optimizer.lr, _lr/10.)
            reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                                          patience=1,
                                          verbose=1)
            #model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=self.fixed_learning_epochs, batch_size=self.batch_size, callbacks=[tensorboard, reduce_lr])
            model.fit(X_train,
                      y_train,
                      validation_data=(X_test, y_test),
                      epochs=self.fixed_learning_epochs,
                      batch_size=self.batch_size,
                      callbacks=[tensorboard])
            _lr = K.get_value(model.optimizer.lr)
            logging.debug(" Current learning rate  (after fit):" + str(_lr))
            scores = model.evaluate(X_test, y_test, verbose=0)

            print(e, scores)
            print("Baseline Error: %.2f%%" % (100 - scores * 100))
            loss = scores

            print("Epoch {:03d}/{:03d} | Loss {:.4f}".format(
                e, self.q_learning_epochs, loss))

            if reference_states is not None:
                q_progress['q_values'].append(
                    self.get_avg_max_q(model,
                                       reference_states=reference_states,
                                       shape=input_shape))
                prev_epochs = q_progress['epochs'][len(q_progress['epochs']) -
                                                   1]
                q_progress['epochs'].append(prev_epochs +
                                            len(self.experience_list))

            # Save trained model weights and architecture, this will be used by the visualization code
            if model_file is not None:
                model.save_weights(model_file, overwrite=True)

        return q_progress
Beispiel #46
0
 def get_config(self):
     config = {'lr': float(K.get_value(self.lr))}
     base_config = super(SGDCust, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
Beispiel #47
0
 def get_config(self):
     config = {'death_rate': K.get_value(self.death_rate)}
     base_config = super().get_config()
     return dict(list(base_config.items()) + list(config.items()))
Beispiel #48
0
 def m_global(self):
     return K.get_value(self.m_global_var)
Beispiel #49
0
    def build(self, input_shape):
        self.input_spec = [InputSpec(shape=input_shape)]
        self.input_dim = input_shape[2]

        if self.stateful:
            self.reset_states()
        else:
            self.states = [None]

        self.W = self.init((self.input_dim, (2 + (not self.coupling)) *
                            self.output_dim), name='{}_W'.format(self.name))
        self.Us = [self.inner_init(
            (self.output_dim, (2 + (not self.coupling)) * self.output_dim),
            name='%s_%d_U' % (self.name, i)) for i in xrange(self.depth)]

        bias_init_value = K.get_value(self.bias_init((self.output_dim,)))
        b = [np.zeros(self.output_dim),
             np.copy(bias_init_value)]

        if not self.coupling:
            b.append(np.copy(bias_init_value))

        self.bs = [K.variable(np.hstack(b),
                              name='%s_%d_b' % (self.name, i)) for i in
                   xrange(self.depth)]

        self.trainable_weights = [self.W] + self.Us + self.bs

        if self.mi:
            self.mi_params = [multiplicative_integration_init(
                ((2 + (not self.coupling)) * self.output_dim,),
                name='%s_%d' % (self.name, i),
                has_input=(i == 0)) for i in xrange(self.depth)]

            for p in self.mi_params:
                if type(p) in {list, tuple}:
                    self.trainable_weights += p
                else:
                    self.trainable_weights += [p]

        if self.has_layer_norm:
            self.ln_weights = []
            ln_names = ['h', 't', 'c']
            for l in xrange(self.depth):

                ln_gains = [self.ln_gain_init(
                    (self.output_dim,), name='%s_%d_ln_gain_%s' %
                    (self.name, l, ln_names[i])) for i in xrange(1)]

                ln_biases = [self.ln_bias_init(
                    (self.output_dim,), name='%s_%d_ln_bias_%s' %
                    (self.name, l, ln_names[i])) for i in xrange(1)]
                self.ln_weights.append([ln_gains, ln_biases])
                self.trainable_weights += ln_gains + ln_biases

        self.regularizers = []
        if self.W_regularizer:
            self.W_regularizer.set_param(self.W)
            self.regularizers.append(self.W_regularizer)
        if self.U_regularizer:
            self.U_regularizer.set_param(self.U)
            self.regularizers.append(self.U_regularizer)
        if self.b_regularizer:
            self.b_regularizer.set_param(self.b)
            self.regularizers.append(self.b_regularizer)

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
Beispiel #50
0
 def k(self):
     return K.get_value(self.k_var)
Beispiel #51
0
def train(BATCH_SIZE, ENC_WEIGHTS, DEC_WEIGHTS):
    print("Loading data...")
    frames_source = hkl.load(os.path.join(DATA_DIR, 'sources_train_128.hkl'))

    # Build video progressions
    videos_list = []
    start_frame_index = 1
    end_frame_index = VIDEO_LENGTH + 1
    while (end_frame_index <= len(frames_source)):
        frame_list = frames_source[start_frame_index:end_frame_index]
        if (len(set(frame_list)) == 1):
            videos_list.append(range(start_frame_index, end_frame_index))
            start_frame_index = start_frame_index + 1
            end_frame_index = end_frame_index + 1
        else:
            start_frame_index = end_frame_index - 1
            end_frame_index = start_frame_index + VIDEO_LENGTH

    videos_list = np.asarray(videos_list, dtype=np.int32)
    n_videos = videos_list.shape[0]

    if SHUFFLE:
        # Shuffle images to aid generalization
        videos_list = np.random.permutation(videos_list)

    # Build the Spatio-temporal Autoencoder
    print("Creating models...")
    encoder = encoder_model()
    decoder = decoder_model()
    autoencoder = autoencoder_model(encoder, decoder)

    run_utilities(encoder, decoder, autoencoder, ENC_WEIGHTS, DEC_WEIGHTS)

    autoencoder.compile(loss='mean_squared_error', optimizer=OPTIM)

    NB_ITERATIONS = int(n_videos / BATCH_SIZE)

    # Setup TensorBoard Callback
    TC = tb_callback.TensorBoard(log_dir=TF_LOG_DIR,
                                 histogram_freq=0,
                                 write_graph=False,
                                 write_images=False)
    LRS = lrs_callback.LearningRateScheduler(schedule=schedule)
    LRS.set_model(autoencoder)

    print("Beginning Training...")
    # Begin Training
    for epoch in range(NB_EPOCHS):
        print("\n\nEpoch ", epoch)
        loss = []

        # Set learning rate every epoch
        LRS.on_epoch_begin(epoch=epoch)
        lr = K.get_value(autoencoder.optimizer.lr)
        print("Learning rate: " + str(lr))

        for index in range(NB_ITERATIONS):
            # Train Autoencoder
            X = load_X(videos_list, index, DATA_DIR)
            # X_train = X[:, 0 : int(VIDEO_LENGTH/2)]
            # y_train = X[:, int(VIDEO_LENGTH/2) :]

            # Change made to generate more output per input frame
            X_train = X[:, 0:10]
            y_train = X[:, 10:]
            loss.append(autoencoder.train_on_batch(X_train, y_train))

            arrow = int(index / (NB_ITERATIONS / 40))
            stdout.write("\rIteration: " + str(index) + "/" +
                         str(NB_ITERATIONS - 1) + "  " + "loss: " +
                         str(loss[len(loss) - 1]) + "\t    [" +
                         "{0}>".format("=" * (arrow)))
            stdout.flush()

        if SAVE_GENERATED_IMAGES:
            # Save generated images to file
            predicted_images = autoencoder.predict(X_train, verbose=0)
            orig_image, truth_image, pred_image = combine_images(
                X_train, y_train, predicted_images)
            pred_image = pred_image * 127.5 + 127.5
            orig_image = orig_image * 127.5 + 127.5
            truth_image = truth_image * 127.5 + 127.5
            if epoch == 0:
                cv2.imwrite(
                    os.path.join(GEN_IMAGES_DIR,
                                 str(epoch) + "_" + str(index) + "_orig.png"),
                    orig_image)
                cv2.imwrite(
                    os.path.join(GEN_IMAGES_DIR,
                                 str(epoch) + "_" + str(index) + "_truth.png"),
                    truth_image)
            cv2.imwrite(
                os.path.join(GEN_IMAGES_DIR,
                             str(epoch) + "_" + str(index) + ".png"),
                pred_image)

        # then after each epoch/iteration
        avg_loss = sum(loss) / len(loss)
        logs = {'loss': avg_loss}
        TC.on_epoch_end(epoch, logs)

        # Log the losses
        with open(os.path.join(LOG_DIR, 'losses.json'), 'a') as log_file:
            log_file.write("{\"epoch\":%d, \"d_loss\":%f};\n" %
                           (epoch, avg_loss))

        print("\nAvg loss: " + str(avg_loss))

        # Save model weights per epoch to file
        encoder.save_weights(
            os.path.join(CHECKPOINT_DIR,
                         'encoder_epoch_' + str(epoch) + '.h5'), True)
        decoder.save_weights(
            os.path.join(CHECKPOINT_DIR,
                         'decoder_epoch_' + str(epoch) + '.h5'), True)

    # End TensorBoard Callback
    TC.on_train_end('_')
Beispiel #52
0
def main(args=None):
    global config
    from keras import backend as K

    # parse arguments
    if args is None:
        args = sys.argv[1:]
    args = parse_args(args)
    print('Arguments: {}'.format(args))

    # create object that stores backbone information
    backbone = models.backbone(args.backbone)

    # make sure keras is the minimum required version
    check_keras_version()

    # optionally choose specific GPU
    if args.gpu:
        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    keras.backend.tensorflow_backend.set_session(get_session())

    # create the generators
    train_generator, validation_generator = create_generators(args, backbone.preprocess_image)

    # create the model
    if args.snapshot is not None:
        print('Loading model, this may take a second...')
        model = models.load_model(args.snapshot, backbone_name=args.backbone)
        training_model = model
        anchor_params = None
        if 'anchor_parameters' in config:
            anchor_params = parse_anchor_parameters(config)
        prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params)
    else:
        weights = args.weights
        # default to imagenet if nothing else is specified
        if weights is None and args.imagenet_weights:
            weights = backbone.download_imagenet()

        print('Creating model, this may take a second...')
        model, training_model, prediction_model = create_models(
            backbone_retinanet=backbone.retinanet,
            num_classes=train_generator.num_classes(),
            weights=weights,
            args=args,
            multi_gpu=args.multi_gpu,
            freeze_backbone=args.freeze_backbone,
            config=config
        )

    # print model summary
    print(model.summary())

    print('Learning rate: {}'.format(K.get_value(model.optimizer.lr)))
    if args.lr > 0.0:
        K.set_value(model.optimizer.lr, args.lr)
        print('Updated learning rate: {}'.format(K.get_value(model.optimizer.lr)))

    # this lets the generator compute backbone layer shapes using the actual backbone model
    if 'vgg' in args.backbone or 'densenet' in args.backbone:
        train_generator.compute_shapes = make_shapes_callback(model)
        if validation_generator:
            validation_generator.compute_shapes = train_generator.compute_shapes

    # create the callbacks
    callbacks = create_callbacks(
        model,
        training_model,
        prediction_model,
        validation_generator,
        args,
    )

    init_epoch = 0
    try:
        if args.snapshot:
            init_epoch = int(args.snapshot.split("_")[-2])
    except:
        pass
    # init_epoch = 6
    print('Init epoch: {}'.format(init_epoch))

    # start training
    training_model.fit_generator(
        generator=train_generator,
        steps_per_epoch=args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,
        initial_epoch=init_epoch,
    )
Beispiel #53
0
 def on_epoch_begin(self, epoch, logs={}):
     print("Learning rate:", K.get_value(model.optimizer.lr))
Beispiel #54
0
        eps = []

        history_train_mse, history_train_angular_errors = [], []
        history_val_mean_angular_errors, history_val_median_angular_errors = [], []
        min_median_angular_error = float('inf')
        for current_epoch in range(1, EPOCHS + 1):
            start_time = timer()
            print('=' * 60)
            print('Epoch {}/{} started.'.format(current_epoch, EPOCHS))

            # learning rate decrease
            if len(history_val_median_angular_errors) > PATIENCE:
                if np.min(history_val_median_angular_errors[-PATIENCE:]
                          ) > history_val_median_angular_errors[-PATIENCE -
                                                                1] - MIN_DELTA:
                    old_lr = float(K.get_value(model.optimizer.lr))
                    new_lr = max(old_lr * FACTOR, MIN_LR)
                    K.set_value(model.optimizer.lr, new_lr)

            # learning rate increase
            if len(history_val_median_angular_errors) > PATIENCE * 10:
                if np.min(
                        history_val_median_angular_errors[-PATIENCE * 10:]
                ) > history_val_median_angular_errors[-PATIENCE * 10 -
                                                      1] - MIN_DELTA:
                    old_lr = float(K.get_value(model.optimizer.lr))
                    new_lr = min(old_lr * 2, MAX_LR)
                    K.set_value(model.optimizer.lr, new_lr)
                    print('Learning rate increased!')

            print('Learning rate in current epoch: {0:.2e}'.format(
Beispiel #55
0
 def on_epoch_begin(self, epoch, logs={}):
     print('EPOCH:', epoch)
     print('\tkl_beta_weight=',
           K.get_value(mod_args['kl_beta_weight']))
     print('\tlr=', float(K.get_value(self.model.optimizer.lr)))
Beispiel #56
0
    # misclassification layer, feed Y
    y_true = Input(shape=(nclass, ), name='y_true')
    psi = mfom.UvZMisclassification(name='uvz_misclass')([y_true, y_pred])

    # class Loss function layer
    out = mfom.SmoothErrorCounter(name='smooth_error_counter')(psi)

    # compile model
    model = Model(input=[y_true, feat_input], output=out)
    model.compile(loss=obj.mfom_eer_normalized, optimizer='Adam')
    model.summary()

    # train
    X, Y = generate_dataset(output_dim=nclass)
    hist = model.fit([Y, X], Y, nb_epoch=100, batch_size=16)

    # calc accuracy: we cut MFoM head, up to sigmoid output
    input = model.get_layer(name='main_input').output
    out = model.get_layer(name='output').output
    cut_model = Model(input=input, output=out)
    y_pred = cut_model.predict(X)
    eer_val = MT.eer(y_true=Y.flatten(), y_pred=y_pred.flatten())
    print('EER: %.4f' % eer_val)

    # history plot, alpha and beta params
    m = model.get_layer('smooth_error_counter')
    print('alpha: ', K.get_value(m.alpha))
    print('beta: ', K.get_value(m.beta))
    plt.plot(hist.history['loss'])
    plt.show()
 def on_epoch_end(self, epoch, logs=None):
     logs = logs or {}
     logs['lr'] = K.get_value(self.model.optimizer.lr)
Beispiel #58
0
def scheduler(epoch):
    K.set_value(vae.optimizer.lr, LearningRate[epoch - 1])
    return float(K.get_value(vae.optimizer.lr))
Beispiel #59
0
def main():
    """Train DeeperGoogLeNet
    """
    # construct the argument parse and parse the arguments
    args = argparse.ArgumentParser()
    args.add_argument("-c", "--checkpoints", required=True, help="path to output checkpoint directory")
    args.add_argument("-m", "--model", type=str, help="path to *specific* model checkpoint to load")
    args.add_argument("-s", "--start-epoch", type=int, default=0, help="epoch to restart training at")
    args = vars(args.parse_args())

    # construct the training image generator for data augmentation
    augmentation = ImageDataGenerator(
        rotation_range=18,
        zoom_range=0.15,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.15,
        horizontal_flip=True,
        fill_mode="nearest",
    )

    # load the RGB means for the training set
    means = json.loads(open(config.DATASET_MEAN).read())
    # initialize the image preprocessors
    simple_preprocessor = SimplePreprocessor(64, 64)
    mean_preprocessor = MeanPreprocessor(means["R"], means["G"], means["B"])
    image_to_array_preprocessor = ImageToArrayPreprocessor()

    # initialize the training and validation dataset generators
    train_gen = HDF5DatasetGenerator(
        config.TRAIN_HDF5,
        64,
        augmentation=augmentation,
        preprocessors=[simple_preprocessor, mean_preprocessor, image_to_array_preprocessor],
        classes=config.NUM_CLASSES,
    )

    val_gen = HDF5DatasetGenerator(
        config.VAL_HDF5,
        64,
        preprocessors=[simple_preprocessor, mean_preprocessor, image_to_array_preprocessor],
        classes=config.NUM_CLASSES,
    )

    # if there is no specific model checkpoint supplied,
    # then initialize the network and compile the model
    if args["model"] is None:
        print("[INFO] compiling model...")
        model = DeeperGoogLeNet.build(width=64, height=64, depth=3, classes=config.NUM_CLASSES, reg=0.0002)
        opt = Adam(1e-3)
        model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"])

    # otherwise, load the checkpoint from disk
    else:
        print("[INFO] loading {}...".format(args["model"]))
        model = load_model(args["model"])
        # update the learning rate
        print("[INFO] old learning rate: {}".format(K.get_value(model.optimizer.lr)))
        K.set_value(model.optimizer.lr, 1e-5)
        print("[INFO] new learning rate: {}".format(K.get_value(model.optimizer.lr)))

    # construct the set of callbacks
    callbacks = [
        EpochCheckpoint(args["checkpoints"], every=5, start_at=args["start_epoch"]),
        TrainingMonitor(config.FIG_PATH, json_path=config.JSON_PATH, start_at=args["start_epoch"]),
    ]

    # train the network
    model.fit_generator(
        train_gen.generator(),
        steps_per_epoch=train_gen.num_images // 64,
        validation_data=val_gen.generator(),
        validation_steps=val_gen.num_images // 64,
        epochs=10,
        max_queue_size=10,
        callbacks=callbacks,
        verbose=1,
    )

    # close the databases
    train_gen.close()
    val_gen.close()
Beispiel #60
0
    def test_save_model_with_dynamic_loss_scaling(
            self, strategy_fn, h5=False, use_v1_loss_scale_optimizer=False):
        # TODO(reedwm): Support and test saving model with a mixed_[b]float16 policy
        # as well.
        strategy = strategy_fn()
        if (isinstance(strategy, tf.distribute.MirroredStrategy)
                and not tf.executing_eagerly()):
            # TODO(b/121381184): Enable running the test in this case.
            return

        # Create and run model.
        with strategy.scope():
            x = layers.Input(shape=(2, ), batch_size=2, dtype=tf.float32)
            y = mp_test_util.MultiplyLayer()(x)
            model = models.Model(inputs=x, outputs=y)

            opt = gradient_descent.SGD(1.)
            if use_v1_loss_scale_optimizer:
                loss_scale = tf.mixed_precision.experimental.DynamicLossScale(
                    initial_loss_scale=1., increment_period=2.)
                opt = loss_scale_optimizer.LossScaleOptimizerV1(
                    opt, loss_scale)
            else:
                opt = loss_scale_optimizer.LossScaleOptimizer(
                    opt, initial_scale=1., dynamic_growth_steps=2.)
            model.compile(optimizer=opt,
                          loss='mse',
                          run_eagerly=testing_utils.should_run_eagerly())
        # Run for 3 steps (6 examples with a batch size of 2)
        model.fit(np.ones((6, 2)), np.zeros((6, 2)), batch_size=2)
        self.assertEqual(backend.get_value(opt.loss_scale), 2)
        self.assertEqual(backend.get_value(opt.dynamic_counter), 1)
        (weight, ) = model.trainable_weights
        orig_weight = backend.get_value(weight)

        # Save model weights.
        save_path = os.path.join(self.get_temp_dir(), 'model')
        model.save(save_path, save_format='h5' if h5 else 'tf')

        # Run model again for 1 step (2 examples with a batch size of 2)
        model.fit(np.ones((2, 2)), np.zeros((2, 2)), batch_size=2)
        new_weight = backend.get_value(weight)
        self.assertNotEqual(new_weight, orig_weight)
        self.assertEqual(backend.get_value(opt.loss_scale), 4)
        self.assertEqual(backend.get_value(opt.dynamic_counter), 0)

        # Load model weights and ensure loss scale weights are restored.
        model = save.load_model(
            save_path,
            custom_objects={'MultiplyLayer': mp_test_util.MultiplyLayer})
        (weight, ) = model.trainable_weights
        loaded_weight = backend.get_value(weight)
        self.assertEqual(loaded_weight, orig_weight)
        # Currently the loss scale isn't always saved when the model is saved with
        # Model.save(). So we assert the loss scale either has the value when it was
        # saved, or the value it was initialized with.
        # TODO(reedwm): Always save/restore the loss scale with Model.save().
        self.assertIn(backend.get_value(model.optimizer.loss_scale), (1, 2))
        self.assertIn(backend.get_value(model.optimizer.dynamic_counter),
                      (0, 1))

        # Test optimizer attributes and type
        self.assertEqual(model.optimizer.initial_scale, 1.)
        self.assertEqual(model.optimizer.dynamic_growth_steps, 2.)
        self.assertEqual(type(model.optimizer),
                         loss_scale_optimizer.LossScaleOptimizer)