Пример #1
0
        # Load dataset
        X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
        X_train = pre_process_images(X_train)
        X_val = pre_process_images(X_val)
        Y_train = one_hot_encode(Y_train, 10)
        Y_val = one_hot_encode(Y_val, 10)

        model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                             use_improved_weight_init)
        trainer = SoftmaxTrainer(
            momentum_gamma,
            use_momentum,
            model,
            learning_rate,
            batch_size,
            shuffle_data,
            X_train,
            Y_train,
            X_val,
            Y_val,
        )
        current_train_history, current_val_history = trainer.train(num_epochs)
        train_history[model_name] = current_train_history
        val_history[model_name] = current_val_history

    plt.figure(figsize=(16, 10))
    plt.subplot(1, 2, 1)
    plt.ylim([0, .5])
    for model_name in train_history.keys():
        utils.plot_loss(train_history[model_name]["loss"],
                        model_name,
Пример #2
0
    # Load dataset
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    # 32 neurons per layer
    model_a = SoftmaxModel(
        neurons_per_layer,
        use_improved_sigmoid,
        use_improved_weight_init)
    trainer_a = SoftmaxTrainer(
        momentum_gamma, use_momentum,
        model_a, learning_rate, batch_size, shuffle_data,
        X_train, Y_train, X_val, Y_val,
    )
    train_history_a, val_history_a = trainer_a.train(num_epochs)

    # 128 neurons per layer
    neurons_per_layer = [128, 10]
    model_b = SoftmaxModel(
        neurons_per_layer,
        use_improved_sigmoid,
        use_improved_weight_init)
    trainer_b = SoftmaxTrainer(
        momentum_gamma, use_momentum,
        model_b, learning_rate, batch_size, shuffle_data,
        X_train, Y_train, X_val, Y_val,
    )
Пример #3
0
    # Load dataset
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    # Example created in assignment text - Comparing with and without shuffling.

    # FIRST CASE (weights)
    use_improved_weight_init = True

    model_weights = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                                 use_improved_weight_init)
    trainer_weights = SoftmaxTrainer(
Пример #4
0
        plt.xlabel("Training steps")
        plt.legend(loc="lower right")

    # First nothing
    use_improved_weight_init = False
    use_improved_sigmoid = False
    use_momentum = False

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )

    train_history_nothing, val_history_nothing = trainer.train(num_epochs)

    # Adding improved weights
    use_improved_weight_init = True
    use_improved_sigmoid = False
    use_momentum = False

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
Пример #5
0
    # Load dataset
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    # Example created in assignment text - Comparing with and without shuffling.
    # YOU CAN DELETE EVERYTHING BELOW!
    use_improved_weight_init = True
    """
    model_improved_weights = SoftmaxModel(
        neurons_per_layer,
        use_improved_sigmoid,
        use_improved_weight_init)
Пример #6
0
    # Load dataset
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    ######Naked model - no improvments######
    model_naked = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                               use_improved_weight_init)
    trainer_naked = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model_naked,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history_naked, val_history_naked = trainer_naked.train(num_epochs)

    print("just basic")
    print("Train accuracy:", calculate_accuracy(X_train, Y_train, model_naked))
    print("Validation accuracy:", calculate_accuracy(X_val, Y_val,
                                                     model_naked))
    print("Final Validation Cross Entropy Loss:",
          cross_entropy_loss(Y_val, model_naked.forward(X_val)))
    print("Final Train Cross Entropy Loss:",
          cross_entropy_loss(Y_train, model_naked.forward(X_train)))
Пример #7
0
    mean = np.mean(X_train)
    std = np.std(X_train)
    X_train = pre_process_images(X_train, mean, std)
    X_val = pre_process_images(X_val, mean, std)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model = SoftmaxModel(neurons_per_layer_small, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        early_stopping,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    # Example created in assignment text - Comparing with and without shuffling.
    # YOU CAN DELETE EVERYTHING BELOW!
    shuffle_data = False
    model_large = SoftmaxModel(neurons_per_layer_large, use_improved_sigmoid,
                               use_improved_weight_init)
    trainer_shuffle = SoftmaxTrainer(
        momentum_gamma,
Пример #8
0
    # Load dataset
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    print("Training standard model:\n")
    model = SoftmaxModel(
        neurons_per_layer,
        use_improved_sigmoid,
        use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma, use_momentum,
        model, learning_rate, batch_size, shuffle_data,
        X_train, Y_train, X_val, Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    print("Final Train Cross Entropy Loss:",
        cross_entropy_loss(Y_train, model.forward(X_train)))
    print("Final Validation Cross Entropy Loss:",
        cross_entropy_loss(Y_val, model.forward(X_val)))
    print("Train accuracy:", calculate_accuracy(X_train, Y_train, model))
    print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
    print("\n\n")

    # Example created in assignment text - Comparing with and without shuffling.
    # YOU CAN DELETE EVERYTHING BELOW!
Пример #9
0
    # Load dataset
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model_previous = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                                  use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model_previous,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    # Example created in assignment text - Comparing with and without shuffling.
    # YOU CAN DELETE EVERYTHING BELOW!
    #Here you edit the new one!
    use_improved_weight_init = True
    use_improved_sigmoid = True
    use_momentum = True
    learning_rate = .02
    neurons_per_layer = [64, 64, 10]
Пример #10
0
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    # Task 4 point a and b
    # Here is presented the model retrieved from the previous task3
    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    print(
        f'Training the Original model {neurons_per_layer[0]} neurons in hidden layer'
    )
    train_history, val_history = trainer.train(num_epochs)

    # Task 4
    # TASK 4 - a) 32 neurons for hidden layer
    neurons_per_layer = [32, 10]
    model_32neu = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                               use_improved_weight_init)
Пример #11
0
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    X_train = pre_process_images(X_train)
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    neurons_per_layer = [64, 10]

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history_64, val_history_64 = trainer.train(num_epochs)
    print("\n\n --------------------------")
    print(neurons_per_layer)
    print("Final Train Cross Entropy Loss:",
          cross_entropy_loss(Y_train, model.forward(X_train)))
    print("Final Validation Cross Entropy Loss:",
          cross_entropy_loss(Y_val, model.forward(X_val)))
    print("Train accuracy:", calculate_accuracy(X_train, Y_train, model))
    print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
    print("\n\n --------------------------")
Пример #12
0
    mean = np.mean(X_train)
    std = np.std(X_train)
    X_train = pre_process_images(X_train, mean, std)
    X_val = pre_process_images(X_val, mean, std)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    # 64 hidden units
    model64 = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                           use_improved_weight_init)
    trainer64 = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model64,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history64, val_history64 = trainer64.train(num_epochs)

    # 32 hidden units
    neurons_per_layer = [32, 10]
    model32 = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                           use_improved_weight_init)
    trainer32 = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model32,
Пример #13
0
    # Task 4e)        #
    ###################
    shuffle_data = True
    use_improved_sigmoid = True
    use_improved_weight_init = True
    use_momentum = False
    learning_rate_momentum = 0.02
    neurons_per_layer = [64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 10]
    model_64_layers = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                                   use_improved_weight_init)
    trainer_shuffle = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model_64_layers,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history_64_layers, val_history_64_layers = trainer_shuffle.train(
        num_epochs)

    plt.figure(figsize=(20, 12))
    plt.subplot(1, 2, 1)

    plt.ylim([0.00, 1.00])
    utils.plot_loss(train_history_64_layers["loss"],
                    "Training Loss 10 hidden layers",
                    npoints_to_average=10)
Пример #14
0
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    mean = np.mean(X_train)
    std = np.std(X_train)
    X_train = pre_process_images(X_train, mean, std)
    X_val = pre_process_images(X_val, mean, std)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    # Example created in assignment text - Comparing with and without shuffling.
    # YOU CAN DELETE EVERYTHING BELOW!
    use_improved_weight_init = True
    use_improved_sigmoid = True
    use_momentum = True
    learning_rate = 0.02

    new_model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
Пример #15
0
        plt.ylim([0.85, 1.01])
        plt.ylabel("Accuracy")
        plt.xlabel("Training steps")
        plt.legend(loc="lower right")

    # Single hidden layer
    neurons_per_layer = [64, 10]

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )

    train_history_single, val_history_single = trainer.train(num_epochs)
    print("---------", neurons_per_layer, "----------")
    print("Final Train Cross Entropy Loss:",
          cross_entropy_loss(Y_train, model.forward(X_train)))
    print("Final Validation Cross Entropy Loss:",
          cross_entropy_loss(Y_val, model.forward(X_val)))
    print("Train accuracy:", calculate_accuracy(X_train, Y_train, model))
    print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
Пример #16
0
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    mean = X_train.mean()
    sd = X_train.std()
    X_train = pre_process_images(X_train, mean, sd)
    X_val = pre_process_images(X_val, mean, sd)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    #two hidden layers
    neurons_per_layer = [59, 59, 10]
    model_two_hidden_layers = SoftmaxModel(neurons_per_layer,
                                           use_improved_sigmoid,
                                           use_improved_weight_init)
    trainer_two_hidden_layers = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
Пример #17
0
    X_val = pre_process_images(X_val)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    ######1nd model - network from task 3######
    use_improved_sigmoid = True
    use_improved_weight_init = True
    use_momentum = True
    model = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                         use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model,
        learning_rate,
        batch_size,
        shuffle_data,
        X_train,
        Y_train,
        X_val,
        Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    ######2nd model - network from task 4d ######
    neurons_per_layer = [60, 60, 10]
    model1 = SoftmaxModel(neurons_per_layer, use_improved_sigmoid,
                          use_improved_weight_init)
    trainer1 = SoftmaxTrainer(
        momentum_gamma,
        use_momentum,
        model1,
Пример #18
0
    X_train, Y_train, X_val, Y_val = utils.load_full_mnist()
    mean = np.mean(X_train)
    std = np.std(X_train)
    X_train = pre_process_images(X_train, mean, std)

    X_val = pre_process_images(X_val, mean, std)
    Y_train = one_hot_encode(Y_train, 10)
    Y_val = one_hot_encode(Y_val, 10)

    model = SoftmaxModel(
        neurons_per_layer,
        use_improved_sigmoid,
        use_improved_weight_init)
    trainer = SoftmaxTrainer(
        momentum_gamma, use_momentum,
        model, learning_rate, batch_size, shuffle_data, early_stop,
        X_train, Y_train, X_val, Y_val,
    )
    train_history, val_history = trainer.train(num_epochs)

    ## RUN WITH MOMENTUM
    use_momentum = True
    model_momentum = SoftmaxModel(
        neurons_per_layer,
        use_improved_sigmoid,
        use_improved_weight_init)
    trainer_momentum = SoftmaxTrainer(
        momentum_gamma, use_momentum,
        model_momentum, learning_rate, batch_size, shuffle_data, early_stop,
        X_train, Y_train, X_val, Y_val,
    )