def compare_optimizers(
    meta_dataset: MetaLearnerDataset,
    optimizer_factories: List[Callable[[np.array, np.array], Optimizer]],
    n_learner_batches: int,
    learner_batch_size: int,
    learner: Model,
    trainings_per_dataset: int,
    initial_learner_weights: Optional[List[np.ndarray]] = None
) -> List[List[float]]:
    """
    Compares performance of two or more optimizers on meta-valid set
    :param meta_dataset: MetaLearnerDataset to get data from
    :param optimizer_factories: list of functions that generate Optimizers to compare
    :param n_learner_batches: number of training batches for a single Learner
    :param learner_batch_size: batch size of Learner
    :param learner: model for Learner
    :param trainings_per_dataset: number of trainings per single dataset per lr value
    :param initial_learner_weights: initial weights for training Learner
    :return: List of Lists of all acquired valid. losses using optimizers on meta-valid tasks
    """
    losses = [[] for _ in optimizer_factories]

    prg_bar = tqdm(total=len(meta_dataset.meta_test_set *
                             trainings_per_dataset * len(optimizer_factories)),
                   desc='Evaluating optimizers...')

    for learner_dataset in meta_dataset.meta_test_set:
        valid_batch_x, valid_batch_y = learner_dataset.test_set.x, learner_dataset.test_set.y
        train_generator = learner_dataset.train_set.batch_generator(
            batch_size=learner_batch_size, randomize=True)

        for _ in range(trainings_per_dataset):
            training_batches = list(islice(train_generator, n_learner_batches))
            if initial_learner_weights is None:
                reset_weights(learner)
                current_initial_learner_weights = learner.get_weights()
            for i, optimizer_factory in enumerate(optimizer_factories):
                # use same batches and initial weights for all optimizers
                learner.optimizer = optimizer_factory(
                    learner_dataset.train_set.x, learner_dataset.train_set.y)
                if initial_learner_weights is None:
                    # noinspection PyUnboundLocalVariable
                    learner.set_weights(current_initial_learner_weights)
                else:
                    learner.set_weights(initial_learner_weights)
                learner.fit_generator(generator=(b for b in training_batches),
                                      steps_per_epoch=n_learner_batches,
                                      epochs=1,
                                      verbose=0)
                evaluation = learner.evaluate(valid_batch_x,
                                              valid_batch_y,
                                              verbose=0)
                if isinstance(evaluation, list):
                    evaluation = evaluation[0]
                losses[i].append(evaluation)
                prg_bar.update(1)

    prg_bar.close()

    return losses
Пример #2
0
    x = conv(x, nb_classes, 1, wd, 0)
    _, r, c, f = x.get_shape().as_list()
    x = Reshape((-1, nb_classes))(x)
    return Activation('softmax')(x)


#Training

input_shape = (224, 224, 3)
img_input = Input(shape=input_shape)
x = create_tiramisu(12,
                    img_input,
                    nb_layers_per_block=[4, 5, 7, 10, 12, 15],
                    p=0.2,
                    wd=1e-4)
model = Model(img_input, x)
gen = segm_generator(x_train, y_train, 3, train=True)
gen_test = segm_generator(x_test, y_test, 3, train=False)
model.compile(loss='sparse_categorical_crossentropy',
              optimizer=keras.optimizers.RMSprop(1e-3),
              metrics=["accuracy"])
model.optimizer = keras.optimizers.RMSprop(1e-3, decay=1 - 0.99995)
model.optimizer = keras.optimizers.RMSprop(1e-3)
K.set_value(model.optimizer.lr, 1e-3)
model.fit_generator(gen,
                    len(y_train),
                    100,
                    verbose=2,
                    validation_data=gen_test,
                    nb_val_samples=len(y_test))
def learning_rate_grid_search(
        optimizer_factory: Callable[[float], Optimizer],
        meta_dataset: MetaLearnerDataset,
        lr_values: List[float],
        n_learner_batches: int,
        learner_batch_size: int,
        learner: Model,
        trainings_per_dataset: int,
        initial_learner_weights: Optional[List[np.ndarray]] = None) -> float:
    """
    Performs grid-search on meta-train set to find best learning rate of optimizer and saves results on valid. set
    :param optimizer_factory: method that returns Optimizer for a given learning rate
    :param meta_dataset: MetaLearnerDataset to get data from
    :param lr_values: list of all values of learning rate to be tested
    :param n_learner_batches: number of training batches for a single Learner
    :param learner_batch_size: batch size of Learner
    :param learner: model for Learner
    :param trainings_per_dataset: number of trainings per single dataset per lr value
    :param initial_learner_weights: initial weights for training Learner
    :return: best lr value
    """
    assert len(lr_values) > 0

    best_lr = 0.0
    best_loss = math.inf

    prg_bar = tqdm(total=len(lr_values) * len(meta_dataset.meta_train_set) *
                   trainings_per_dataset,
                   desc='Grid-Search of learning rate')

    for lr in lr_values:
        total_loss = 0.0

        for learner_dataset in meta_dataset.meta_train_set:
            valid_batch_x, valid_batch_y = learner_dataset.test_set.x, learner_dataset.test_set.y
            for _ in range(trainings_per_dataset):
                learner.optimizer = optimizer_factory(lr)
                if initial_learner_weights is None:
                    reset_weights(learner)
                else:
                    learner.set_weights(initial_learner_weights)
                if isinstance(learner.optimizer, Model):
                    learner.optimizer.reset_states()
                learner.fit_generator(
                    generator=learner_dataset.train_set.batch_generator(
                        batch_size=learner_batch_size, randomize=True),
                    steps_per_epoch=n_learner_batches,
                    epochs=1,
                    verbose=0)
                evaluation = learner.evaluate(valid_batch_x,
                                              valid_batch_y,
                                              verbose=0)
                if isinstance(evaluation, list):
                    evaluation = evaluation[0]
                total_loss += evaluation
                prg_bar.update(1)

        if total_loss < best_loss:
            best_loss = total_loss
            best_lr = lr

    prg_bar.close()

    return best_lr
def analyze_training(
        meta_dataset: MetaLearnerDataset,
        optimizer_factory: Callable[[np.array, np.array], Optimizer],
        n_learner_batches: int,
        learner_batch_size: int,
        learner: Model,
        trainings_per_dataset: int,
        initial_learner_weights: Optional[List[np.ndarray]] = None):
    """
    Analyze statistics during training Learner using Meta-Learner
    :param meta_dataset: MetaLearnerDataset to get data from
    :param optimizer_factory: functions that generates Optimizer
    :param n_learner_batches: number of training batches for a single Learner
    :param learner_batch_size: batch size of Learner
    :param learner: model for Learner
    :param trainings_per_dataset: number of trainings per single dataset per lr value
    :param initial_learner_weights: initial weights for training Learner
    :return: Tuple of lists of statistics
    """
    train_losses, train_accuracies = [], []
    valid_losses, valid_accuracies = [], []
    hessian_eigen = []

    eigenvals_callback = TopKEigenvaluesBatched(K=4,
                                                batch_size=learner_batch_size,
                                                logger=None,
                                                save_dir="",
                                                save_eigenv=1)
    eigenvals_callback.model = learner
    eigenvals_callback.compile()

    prg_bar = tqdm(total=len(meta_dataset.meta_test_set *
                             trainings_per_dataset),
                   desc='Analyzing Learner trainings...')

    for learner_dataset in meta_dataset.meta_test_set:
        train_x, train_y = learner_dataset.train_set.x, learner_dataset.train_set.y
        valid_x, valid_y = learner_dataset.test_set.x, learner_dataset.test_set.y

        eigenvals_callback.X = train_x
        eigenvals_callback.y = train_y

        for _ in range(trainings_per_dataset):
            current_train_losses, current_train_accuracies = [], []
            current_valid_losses, current_valid_accuracies = [], []
            current_eigen = []

            if initial_learner_weights is None:
                reset_weights(learner)
            else:
                learner.set_weights(initial_learner_weights)

            learner.optimizer = optimizer_factory(train_x, train_y)

            for i, training_batch in enumerate(
                    learner_dataset.train_set.batch_generator(
                        batch_size=learner_batch_size, randomize=True)):
                if i >= n_learner_batches:
                    break
                learner.train_on_batch(training_batch[0], training_batch[1])

                train_evaluation = learner.evaluate(train_x,
                                                    train_y,
                                                    verbose=0)
                assert isinstance(train_evaluation, list)
                current_train_losses.append(train_evaluation[0])
                current_train_accuracies.append(train_evaluation[1])

                valid_evaluation = learner.evaluate(valid_x,
                                                    valid_y,
                                                    verbose=0)
                assert isinstance(valid_evaluation, list)
                current_valid_losses.append(valid_evaluation[0])
                current_valid_accuracies.append(valid_evaluation[1])

                eigen = np.mean(
                    eigenvals_callback.compute_top_K(with_vectors=False)[:-1])
                current_eigen.append(eigen)

            train_losses.append(current_train_losses)
            train_accuracies.append(current_train_accuracies)
            valid_losses.append(current_valid_losses)
            valid_accuracies.append(current_valid_accuracies)
            hessian_eigen.append(current_eigen)

            prg_bar.update(1)

    prg_bar.close()
    print()

    return train_losses, train_accuracies, valid_losses, valid_accuracies, hessian_eigen