Beispiel #1
0
def Franke_plot_overfitting(X_train,
                            X_test,
                            eta=0,
                            lmbd=0,
                            batch_size=0,
                            n_hidden_neurons=0,
                            epochs=0):

    eta = 3.16227766e-01
    lmbd = 2.68269580e-08
    batch_size = 1
    n_hidden_neurons = 57
    epochs = 1000
    n_categories = 1

    np.random.seed(seed)
    dnn = NN(X_train,
             z_train,
             eta=eta,
             lmbd=lmbd,
             epochs=int(epochs),
             batch_size=batch_size,
             n_hidden_neurons=n_hidden_neurons,
             n_categories=n_categories,
             cost_grad='MSE',
             activation='sigmoid',
             activation_out='ELU')
    dnn.train_and_validate(X_val, z_val, MSE_store=True, validate=False)
    MSE_val, MSE_train = dnn.MSE_epoch()
    epo = np.arange(len(MSE_train))
    plt.plot(epo, MSE_val, label='MSE val')
    plt.plot(epo, MSE_train, label='MSE train')
    plt.xlabel("Number of epochs")
    plt.ylabel("MSE")
    plt.title("MSE vs epochs")
    plt.legend()
    plt.show()
Beispiel #2
0
def hypertuning(iterations, cols, etamax, etamin, lmbdmax, lmbdmin,
                batch_sizemax, batch_sizemin, hiddenmax, hiddenmin):
    start_time = time.time()
    if (cols < iterations):
        cols = iterations
        print(
            "cols must be larger than 'iterations. Cols is set equal to iterations"
        )
    sig = signature(hypertuning)
    rows = int(len(sig.parameters) / 2)
    hyper = np.zeros((rows, cols))
    MSE_array = np.zeros(iterations)
    hyper[0] = np.logspace(etamin, etamax, cols)
    hyper[1] = np.logspace(lmbdmin, lmbdmax, cols)
    hyper[2] = np.round(
        np.linspace(batch_sizemin, batch_sizemax, cols, dtype='int'))
    hyper[3] = np.round(np.linspace(hiddenmin, hiddenmax, cols))
    hyper[4] = np.zeros((cols))
    #print(hyper)
    for i in range(rows - 1):
        np.random.shuffle(hyper[i])
        #print(np.apply_along_axis(np.random.shuffle, 1, hyper[i]))

    for it in range(iterations):
        hyper_choice = hyper[:, it]
        eta = hyper_choice[0]
        lmbd = hyper_choice[1]
        batch_size = hyper_choice[2]
        n_hidden_neurons = hyper_choice[3]

        dnn = NN(X_train,
                 z_train,
                 eta=eta,
                 lmbd=lmbd,
                 epochs=epochs,
                 batch_size=batch_size,
                 n_hidden_neurons=n_hidden_neurons,
                 n_categories=n_categories,
                 cost_grad='MSE',
                 activation='sigmoid',
                 activation_out='ELU')

        dnn.train(X_val)
        MSE_val = dnn.MSE_epoch(z_val)

        best_pred_epoch = np.argmin(MSE_val)

        dnn_test = NN(X_train,
                      z_train,
                      eta=eta,
                      lmbd=lmbd,
                      epochs=best_pred_epoch + 1,
                      batch_size=batch_size,
                      n_hidden_neurons=n_hidden_neurons,
                      n_categories=n_categories,
                      cost_grad='MSE',
                      activation='sigmoid',
                      activation_out='ELU')
        dnn_test.train(X_test)

        # kan jo bare bruke predict probabilities på siste her

        z_pred = dnn_test.y_predict_epoch[best_pred_epoch]

        MSE_array[it] = mean_squared_error(z_test, z_pred)
        hyper[4][it] = best_pred_epoch
        print(it)
        if (it % m.ceil((iterations / 40)) == 0):
            t = round((time.time() - start_time))
            if t >= 60:
                sec = t % 60
                print("--- %s min," % int(t / 60), "%s sec ---" % sec)
            else:
                print("--- %s sec ---" % int(t))
    MSE_best_index = np.argmin(MSE_array)
    MSE_best = np.min(MSE_array)
    print("MSE array: ", MSE_array)
    print("best index: ", MSE_best_index)
    print("best MSE: ", MSE_best)

    return hyper[:, MSE_best_index]
Beispiel #3
0
def hypertuning_franke(z, x, y, iterations, cols, etamin, etamax, lmbdmin, lmbdmax, batch_sizemin, batch_sizemax, hiddenmin,hiddenmax, polymin, polymax, epochs = 1000, plot_MSE = False, validate = True):

    start_time = time.time()
    if (cols < iterations):
        cols = iterations
        print("cols must be larger than 'iterations. Cols is set equal to iterations")
    rows = 6
    hyper = np.zeros((rows, cols))
    MSE_array = np.zeros(iterations)
    #Making the matrix of parameters 
    hyper[0] =  np.logspace(etamin, etamax, cols)
    hyper[1] = np.logspace(lmbdmin, lmbdmax, cols)
    hyper[2] = np.round(np.linspace(batch_sizemin, batch_sizemax, cols, dtype='int'))
    hyper[3] = np.round(np.linspace(hiddenmin, hiddenmax, cols))
    hyper[4] = np.random.randint(polymin, polymax, size=cols, dtype='int')
    hyper[5] = np.zeros((cols))
    

    
    for i in range(rows-1):
        np.random.shuffle(hyper[i])

    n_categories = 1
    
    #iterating over all parameters 
    for it in range(iterations):
        hyper_choice = hyper[:,it]
        eta = hyper_choice[0]
        lmbd = hyper_choice[1]
        batch_size = hyper_choice[2]
        n_hidden_neurons = hyper_choice[3]
        X, X_train, X_test, X_val, z_train, z_test, z_val, indicies = CreateDesignMatrix_X(z, x,y, int(hyper[4][it]))

        np.random.seed(seed)
        dnn = NN(X_train, z_train, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, n_hidden_neurons=n_hidden_neurons, n_categories=n_categories,
                 cost_grad='MSE', activation='sigmoid', activation_out='ELU')
        dnn.train_and_validate(X_val, z_val, MSE_store = plot_MSE, validate=validate)

        z_pred = dnn.predict_probabilities(X_val)
        MSE_array[it] = mean_squared_error(z_val,z_pred)
        hyper[5][it] = dnn.epoch +1

        #Optional: If one wishes to see how the parameter combination is doing, pass plot_MSE = True:
        if(plot_MSE):
            print("parameters: eta, lmbd, batch, hidden, poly, epochs \n", hyper[:,it:it+1])
            MSE_val, MSE_train = dnn.MSE_epoch()
            epo = np.arange(len(MSE_val))
            plt.plot(epo, MSE_val, label='MSE val')
            plt.plot(epo, MSE_train, label='MSE train')
            plt.xlabel("Number of epochs")
            plt.ylabel("MSE")
            plt.title("MSE vs epochs")
            plt.legend()
            plt.show()

        #Estimating the time the iteration takes
        if (it%m.ceil((iterations/60))==0):
            print('Iteration: ', it)
            t = round((time.time() - start_time))
            if (t >= 60) and (it > 0):
                sec = t % 60
                print("--- %s min," % int(t/60),"%s sec ---" % sec)
                print("Estimated minutes left: ", int((t/it)*(iterations-it)/60))
            else:
                print("--- %s sec ---" %int(t))

    # Finding the best parameters:
    MSE_best_index = np.argmin(MSE_array)
    MSE_best = np.min(MSE_array)
    print("MSE array: ", MSE_array)
    print("best index: ",MSE_best_index)
    print("best MSE: ", MSE_best)
    final_hyper = hyper[:,MSE_best_index]

    print("parameters: eta, lmbd, batch, hidden, poly, epochs ", final_hyper)
    eta_best = final_hyper[0]
    lmbd_best = final_hyper[1]
    batch_size_best = final_hyper[2]
    n_hidden_neurons_best = final_hyper[3]
    poly_best = final_hyper[4]
    epochs_best = final_hyper[5]
    return hyper[:,MSE_best_index]