Esempio n. 1
0
def grid_search():
	X, Y = get_spiral()
	X, Y = shuffle(X, Y)
	Ntrain = int(0.7*len(X))
	Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
	Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

	hidden_layer_sizes = [[300], [100, 100], [50, 50, 50]]
	learning_rates = [1e-4, 1e-3, 1e-2]
	l2_penalties = [0., 0.1, 1.0]

	best_validation_rate = 0
	best_hls = None
	best_lr = None
	best_l2 = None
	for hls in hidden_layer_sizes:
		for lr in learning_rates:
			for l2 in l2_penalties:
				model = ANN(hls)
				model.fit(Xtrain, Ytrain, learning_rate = lr, reg = l2, mu = 0.99, epochs = 3000, show_fig = False)
				validation_accuracy = model.score(Xtest, Ytest)
				train_accuracy = model.score(Xtrain, Ytrain)

				print("validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s" %
            (validation_accuracy, train_accuracy, hls, lr, l2))
				if validation_accuracy > best_validation_rate:
					best_validation_rate = validation_accuracy
					best_hls = hls
					best_lr = lr
					best_l2 = l2
	print("Best validation_accuracy:", best_validation_rate)
	print("Best settings:")
	print("hidden_layer_sizes:", best_hls)
	print("learning_rate:", best_lr)
	print("l2:", best_l2)
Esempio n. 2
0
def random_search():
    # get the data and split into train/test
    X, Y = get_spiral()
    # X, Y = get_clouds()
    X, Y = shuffle(X, Y)
    Ntrain = int(0.7 * len(X))
    Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
    Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

    # starting hyperparameters
    M = 20
    nHidden = 2
    log_lr = -4
    log_l2 = -2  # since we always want it to be positive
    max_tries = 30

    # loop through all possible hyperparameter settings
    best_validation_rate = 0
    best_hls = None
    best_lr = None
    best_l2 = None
    for _ in range(max_tries):
        model = ANN([M] * nHidden)
        model.fit(Xtrain,
                  Ytrain,
                  learning_rate=10**log_lr,
                  reg=10**log_l2,
                  mu=0.99,
                  epochs=3000,
                  show_fig=False)
        validation_accuracy = model.score(Xtest, Ytest)
        train_accuracy = model.score(Xtrain, Ytrain)
        print(
            "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s"
            % (validation_accuracy, train_accuracy, [M] * nHidden, log_lr,
               log_l2))
        if validation_accuracy > best_validation_rate:
            best_validation_rate = validation_accuracy
            best_M = M
            best_nHidden = nHidden
            best_lr = log_lr
            best_l2 = log_l2

        # select new hyperparams
        nHidden = best_nHidden + np.random.randint(-1, 2)  # -1, 0, or 1
        nHidden = max(1, nHidden)
        M = best_M + np.random.randint(-1, 2) * 10
        M = max(10, M)
        log_lr = best_lr + np.random.randint(-1, 2)
        log_l2 = best_l2 + np.random.randint(-1, 2)
    print("Best validation_accuracy:", best_validation_rate)
    print("Best settings:")
    print("best_M:", best_M)
    print("best_nHidden:", best_nHidden)
    print("learning_rate:", best_lr)
    print("l2:", best_l2)
Esempio n. 3
0
def random_search():

    X, Y = get_clouds()
    X, Y = shuffle(X, Y)

    Ntrain = int(0.7 * len(X))

    Xtrain, Xtest = X[:Ntrain], X[Ntrain:]
    Ytrain, Ytest = Y[:Ntrain], Y[Ntrain:]

    M = 20
    nHidden = 2
    log_lr = -4
    log_l2 = -2
    max_tries = 30
    best_nHidden = None
    best_validation_rate = 0
    best_M = None
    best_lr = None
    best_l2 = None
    for _ in range(max_tries):
        model = ANN([M] * nHidden)
        model.fit(Xtrain,
                  Ytrain,
                  learning_rate=10**log_lr,
                  reg=10**log_l2,
                  mu=0.99,
                  epochs=3000,
                  show_fig=False)
        validation_accuracy = model.score(Xtest, Ytest)
        train_accuracy = model.score(Xtrain, Ytrain)
        print(
            f'validation_accuracy: {validation_accuracy}, train_accuracy: {train_accuracy}, setting M: {M}, nHidden: {nHidden}, lr: {log_lr}, l2: {log_l2}'
        )

        if validation_accuracy > best_validation_rate:
            best_validation_rate = validation_accuracy
            best_M = M
            best_nHidden = nHidden
            best_l2 = log_l2
            best_lr = log_lr

        # select new hyperparams
        nHidden = best_nHidden + np.random.randint(-1, 2)
        nHidden = max(1, nHidden)
        M = best_M + np.random.randint(-1, 2) * 10
        M = max(10, M)
        log_lr = best_lr + np.random.randint(-1, 2)
        log_l2 = best_l2 + np.random.randint(-1, 2)

    print(f'Best validation_accuracy: {best_validation_rate}')
    print('Best Setting:')
    print(f'Best hidden layer number: {best_nHidden}')
    print(f"Best hidden_layer_size, {best_M}")
    print(f'Best learning rate: {best_lr}')
    print(f'Best regularizations : {best_l2}')
def random_search():
  # get the data and split into train/test
  X, Y = get_spiral()
  # X, Y = get_clouds()
  X, Y = shuffle(X, Y)
  Ntrain = int(0.7*len(X))
  Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
  Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

  # starting hyperparameters
  M = 20
  nHidden = 2
  log_lr = -4
  log_l2 = -2 # since we always want it to be positive
  max_tries = 30
  

  # loop through all possible hyperparameter settings
  best_validation_rate = 0
  best_hls = None
  best_lr = None
  best_l2 = None
  for _ in range(max_tries):
    model = ANN([M]*nHidden)
    model.fit(
      Xtrain, Ytrain,
      learning_rate=10**log_lr, reg=10**log_l2,
      mu=0.99, epochs=3000, show_fig=False
    )
    validation_accuracy = model.score(Xtest, Ytest)
    train_accuracy = model.score(Xtrain, Ytrain)
    print(
      "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s" %
        (validation_accuracy, train_accuracy, [M]*nHidden, log_lr, log_l2)
    )
    if validation_accuracy > best_validation_rate:
      best_validation_rate = validation_accuracy
      best_M = M
      best_nHidden = nHidden
      best_lr = log_lr
      best_l2 = log_l2

    # select new hyperparams
    nHidden = best_nHidden + np.random.randint(-1, 2) # -1, 0, or 1
    nHidden = max(1, nHidden)
    M = best_M + np.random.randint(-1, 2)*10
    M = max(10, M)
    log_lr = best_lr + np.random.randint(-1, 2)
    log_l2 = best_l2 + np.random.randint(-1, 2)
  print("Best validation_accuracy:", best_validation_rate)
  print("Best settings:")
  print("best_M:", best_M)
  print("best_nHidden:", best_nHidden)
  print("learning_rate:", best_lr)
  print("l2:", best_l2)
Esempio n. 5
0
def main():
	X, Y = get_spiral()
	N, D = X.shape
	num_train = int(0.7*N)
	train_X = X[:num_train, :]
	train_Y = Y[:num_train]
	test_X = X[num_train:, :]
	test_Y = Y[num_train:]

	#initial hyperparameters
	M = 20
	num_hidden_layers = 2
	log_lr = -4
	log_reg = -2
	max_iter = 30

	best_M = M
	best_num_hidden_layers = num_hidden_layers
	best_validation_rate = 0
	best_hidden_layer_size = None
	best_learning_rate = None
	best_reg_penalty = None

	for i in range(max_iter):
		model = ANN([M] * num_hidden_layers)
		model.fit(train_X, train_Y, learning_rate=10**log_lr, reg=10**log_reg, mu=0.99, epochs=3000, show_fig=False)
		validation_rate = model.score(test_X, test_Y)
		train_accuracy = model.score(train_X, train_Y)
		print("validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s" % (validation_rate, train_accuracy, [M]*num_hidden_layers, log_lr, log_reg))

		if validation_rate > best_validation_rate:
			best_M = M
			best_num_hidden_layers = num_hidden_layers
			best_validation_rate = validation_rate
			best_hidden_layer_size = [M] * num_hidden_layers
			best_learning_rate = log_lr
			best_reg_penalty = log_reg

		#update
		num_hidden_layers = best_num_hidden_layers + np.random.randint(-1, 2)
		M = best_M + np.random.randint(-1, 2)
		num_hidden_layers = max(10, num_hidden_layers)
		M = max(1, M)

		log_lr = best_learning_rate + np.random.randint(-1, 2)
		log_reg = best_reg_penalty + np.random.randint(-1, 2)
		log_lr = min(0, log_lr)
		log_reg = min(0, log_lr)

	print("Best validation_accuracy:", best_validation_rate)
	print("Best settings:")
	print("hidden_layer_sizes:", best_M*best_num_hidden_layers)
	print("learning_rate:", 10**best_learning_rate)
	print("l2:", 10**best_reg_penalty)
Esempio n. 6
0
def random_search():
    X, Y = get_spiral()
    X, Y = shuffle(X, Y)
    Ntrain = int(0.7 * len(X))
    Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
    Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

    #starting parameter
    M = 20
    nHidden = 2
    log_lr = -4
    log_l2 = -2
    max_iters = 30

    best_validation_rate = 0
    best_hls = None
    best_lr = None
    best_l2 = None
    for _ in range(max_iters):
        model = ANN([M] * nHidden)
        model.fit(Xtrain,
                  Ytrain,
                  learning_rate=10**log_lr,
                  reg=10**log_l2,
                  mu=0.99,
                  epochs=3000,
                  show_fig=False)
        validation_accuracy = model.score(Xtest, Ytest)
        train_accuracy = model.score(Xtrain, Ytrain)

        print(
            "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s"
            % (validation_accuracy, train_accuracy, [M] * nHidden, log_lr,
               log_l2))
        if validation_accuracy > best_validation_rate:
            best_validation_rate = validation_accuracy
            best_M = M
            best_nHidden = nHidden
            best_lr = log_lr
            best_l2 = log_l2
        nHidden = best_nHidden + np.random.randint(-1, 2)
        nHidden = max(1, nHidden)
        M = best_M + np.random.randint(-1, 2) * 10
        M = max(10, M)
        log_lr = best_lr + np.random.randint(-1, 2)
        log_l2 = best_l2 + np.random.randint(-1, 2)

    print("Best validation_accuracy:", best_validation_rate)
    print("Best settings:")
    print("best_M:", best_M)
    print("best_nHidden:", best_nHidden)
    print("learning_rate:", best_lr)
    print("l2:", best_l2)
def grid_search():
    # get the data and split into train/test
    X, Y = get_spiral()
    # X, Y = get_clouds()
    X, Y = shuffle(X, Y)
    Ntrain = int(len(X) * 0.7)
    Xtrain, Ytrain = X[:Ntrain, ], Y[:Ntrain, ]
    Xtest, Ytest = X[Ntrain:, ], Y[
        Ntrain:,
    ]  # here is more like a validation set, because we use it to choose hyperparameter settings

    # hyperparameters to try
    hidden_layer_sizes = [
        [300],
        [100, 100],
        [50, 50, 50],
    ]
    learning_rates = [1e-4, 1e-3, 1e-2]
    l2_penalties = [0., 0.1, 1.0]

    # loop through all possible hyperparameter settings
    best_validation_rate = 0
    best_hls = None
    best_lr = None
    best_l2 = None
    for hls in hidden_layer_sizes:
        for lr in learning_rates:
            for l2 in l2_penalties:
                model = ANN(hls)
                model.fit(Xtrain,
                          Ytrain,
                          learning_rate=lr,
                          reg=l2,
                          mu=0.99,
                          epochs=3000,
                          show_fig=False)
                validation_accuracy = model.score(Xtest, Ytest)
                train_accuracy = model.score(Xtrain, Ytrain)
                print(
                    "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s"
                    % (validation_accuracy, train_accuracy, hls, lr, l2))
                if validation_accuracy > best_validation_rate:
                    best_validation_rate = validation_accuracy
                    best_hls = hls
                    best_lr = lr
                    best_l2 = l2
    print("Best validation_accuracy:", best_validation_rate)
    print("Best settings:")
    print("hidden_layer_sizes:", best_hls)
    print("learning_rate:", best_lr)
    print("l2:", best_l2)
Esempio n. 8
0
def grid_search():

    X, Y = get_spiral()
    X, Y = shuffle(X, Y)

    Ntrain = int(0.7 * len(X))

    Xtrain, Xtest = X[:Ntrain], X[Ntrain:]
    Ytrain, Ytest = Y[:Ntrain], Y[Ntrain:]

    hidden_layer_sizes = [[300], [100, 100], [200, 200], [50, 50, 50]]

    learning_rates = [1e-4, 1e-3, 1e-2, 1e-5]

    regularizations = [0., 0.1, 1.0]
    best_validation_rate = 0
    best_hls = None
    best_lr = None
    best_l2 = None
    for m in hidden_layer_sizes:
        for lr in learning_rates:
            for l2 in regularizations:
                model = ANN(m)
                model.fit(Xtrain,
                          Ytrain,
                          learning_rate=lr,
                          reg=l2,
                          mu=0.99,
                          epochs=3000,
                          show_fig=False)
                validation_accuracy = model.score(Xtest, Ytest)
                train_accuracy = model.score(Xtrain, Ytrain)
                print(
                    f'validation_accuracy: {validation_accuracy}, train_accuracy: {train_accuracy}, setting hls,lr,l2: {m}, {lr}, {l2}'
                )

                if validation_accuracy > best_validation_rate:
                    best_validation_rate = validation_accuracy
                    best_hls = m
                    best_l2 = l2
                    best_lr = lr

    print(f'Best validation_accuracy: {best_validation_rate}')
    print('Best Setting:')
    print(f"Best hidden_layer_size, {best_hls}")
    print(f'Best learning rate: {best_lr}')
    print(f'Best regularizations : {best_l2}')
Esempio n. 9
0
def main():
    X, Y = get_spiral()
    N, D = X.shape
    num_train = int(0.7 * N)
    train_X = X[:num_train, :]
    train_Y = Y[:num_train]
    test_X = X[num_train:, :]
    test_Y = Y[num_train:]

    #hyperparamters need to search
    hidden_layer_size = [[300], [100, 100], [50, 50, 50]]
    learning_rate = [1e-4, 1e-3, 1e-2]
    reg_penalties = [0., 0.1, 1.0]

    best_validation_rate = 0
    best_hidden_layer_size = None
    best_learning_rate = None
    best_reg_penalties = None

    for hlz in hidden_layer_size:
        for lr in learning_rate:
            for rp in reg_penalties:
                model = ANN(hlz)
                model.fit(train_X,
                          train_Y,
                          learning_rate=lr,
                          reg=rp,
                          mu=0.99,
                          epochs=3000,
                          show_fig=False)
                validation_rate = model.score(test_X, test_Y)
                train_accuracy = model.score(train_X, train_Y)
                print(
                    "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s"
                    % (validation_rate, train_accuracy, hlz, lr, rp))

                if validation_rate > best_validation_rate:
                    best_validation_rate = validation_rate
                    best_hidden_layer_size = hlz
                    best_learning_rate = lr
                    best_reg_penalties = rp
    print("Best validation_accuracy:", best_validation_rate)
    print("Best settings:")
    print("hidden_layer_sizes:", best_hidden_layer_size)
    print("learning_rate:", best_learning_rate)
    print("l2:", best_reg_penalties)
def grid_search():
  # get the data and split into train/test
  X, Y = get_spiral()
  # X, Y = get_clouds()
  X, Y = shuffle(X, Y)
  Ntrain = int(0.7*len(X))
  Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
  Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

  # hyperparameters to try
  hidden_layer_sizes = [
    [300],
    [100,100],
    [50,50,50],
  ]
  learning_rates = [1e-4, 1e-3, 1e-2]
  l2_penalties = [0., 0.1, 1.0]

  # loop through all possible hyperparameter settings
  best_validation_rate = 0
  best_hls = None
  best_lr = None
  best_l2 = None
  for hls in hidden_layer_sizes:
    for lr in learning_rates:
      for l2 in l2_penalties:
        model = ANN(hls)
        model.fit(Xtrain, Ytrain, learning_rate=lr, reg=l2, mu=0.99, epochs=3000, show_fig=False)
        validation_accuracy = model.score(Xtest, Ytest)
        train_accuracy = model.score(Xtrain, Ytrain)
        print(
          "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s" %
            (validation_accuracy, train_accuracy, hls, lr, l2)
        )
        if validation_accuracy > best_validation_rate:
          best_validation_rate = validation_accuracy
          best_hls = hls
          best_lr = lr
          best_l2 = l2
  print("Best validation_accuracy:", best_validation_rate)
  print("Best settings:")
  print("hidden_layer_sizes:", best_hls)
  print("learning_rate:", best_lr)
  print("l2:", best_l2)
Esempio n. 11
0
def random_search():
    X, Y = get_spiral()
    # X, Y = get_clouds()
    X, Y = shuffle(X, Y)
    Ntrain = int(0.7*len(X))
    Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
    Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

    # starting parameters
    M = 20
    nHidden = 2
    log_lr = -4 # use log values, since typical adjustments are .1, .01...., .000001 etc
    log_l2 = -2
    max_tries = 30

    # loop through all possible hyperparameter settings
    best_validation_rate = 0
    best_hls = None
    best_lr = None
    best_l2 = None
    for _ in range(max_tries):
        model = ANN([M]*nHidden)
        # raise 10 to log_lr and log_l2 power since they're in log scale
        model.fit(Xtrain, Ytrain, learning_rate=10**log_lr, reg=10**log_l2, mu=0.99, epochs=3000, show_fig=False)
        validation_accuracy = model.score(Xtest, Ytest)
        train_accuracy = model.score(Xtrain, Ytrain)
        print("validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s, %s, %s" %
            (validation_accuracy, train_accuracy, [M]*nHidden], log_lr, log_l2)
            )
        if validation_accuracy > best_validation_rate:
            best_validation_rate = validation_accuracy
            best_M = M
            best_nHidden = nHidden
            best_lr = log_lr
            best_l2 = log_l2

        # Select new hyperparameters
        nHidden = best_nHidden * np.random.randint(-1,2) # -1, 0, or 1
        nHidden = max(1,nHidden)
        M = best_M + np.random.randint(-1,2)*10
        M = max(10,M)
        log_lr = best_lr + np.random.randint(-1,2)
        log_l2 = best_l2 + np.random.randint(-1,2)
Esempio n. 12
0
def random_search():

    X, Y, data = get_data()
    X, Y = shuffle(X, Y)
    Ntrain = int(0.75 * len(X))
    Xtrain, Ytrain = X[:Ntrain], Y[:Ntrain]
    Xtest, Ytest = X[Ntrain:], Y[Ntrain:]

    # Make copies of the small data (because variance matters?)
    Xtrain = np.concatenate((Xtrain, Xtrain, Xtrain), 0)
    Ytrain = np.concatenate((Ytrain, Ytrain, Ytrain), 0)
    # TODO: add some noise to the data ... add some noise to the weights (noise injection, should reduce overfitting)
    # Add gaussian noise, small variance. (np.random.randn(N)*0.5) = Gaussian w/ 0.5 variance

    print('size Xtrain: ' + str(Xtrain.shape))
    print('size Ytrain: ' + str(Ytrain.shape))
    print('size Xtest: ' + str(Xtest.shape))
    print('size Ytest: ' + str(Ytest.shape))

    # starting hyperparameters
    M = 20  # hidden units
    nHidden = 2  # hidden layers
    log_lr = -4  # learning rate
    log_l2 = -2  # l2 regularization, since we always want it to be positive
    max_tries = 30

    # loop through all possible hyperparameter settings
    best_validation_rate = 0
    best_hls = None
    best_lr = None
    best_l2 = None
    validation_accuracies = []
    for _ in range(max_tries):
        print('on try: ' + str(_ + 1) + '/' + str(max_tries))
        model = ANN([M] * nHidden)
        # choose params randomly on log base 10 scale
        model.fit(Xtrain,
                  Ytrain,
                  learning_rate=10**log_lr,
                  reg=10**log_l2,
                  mu=0.99,
                  epochs=4000,
                  show_fig=False)
        validation_accuracy = model.score(Xtest, Ytest)
        train_accuracy = model.score(Xtrain, Ytrain)
        print(
            "validation_accuracy: %.3f, train_accuracy: %.3f, settings: %s (layers), %s (log_lr), %s (log_l2)"
            % (validation_accuracy, train_accuracy, [M] * nHidden, log_lr,
               log_l2))
        # keep track of all
        validation_accuracies.append(validation_accuracy)

        # keep the best parameters, then make modifications to them
        if validation_accuracy > best_validation_rate:
            best_validation_rate = validation_accuracy
            best_M = M
            best_nHidden = nHidden
            best_lr = log_lr
            best_l2 = log_l2

        # select new hyperparams
        nHidden = best_nHidden + np.random.randint(
            -1, 2)  # -1, 0, or 1, add, remove or keep same the layers
        nHidden = max(1, nHidden)
        M = best_M + np.random.randint(-1, 2) * 10
        M = max(10, M)
        log_lr = best_lr + np.random.randint(-1, 2)
        log_l2 = best_l2 + np.random.randint(-1, 2)

    # TODO: save these in mongodb, then read them and see if we beat it, in a new file run forward on best params
    print("Best validation_accuracy:", best_validation_rate)
    print("Mean validation_accuracy:", np.mean(validation_accuracies))
    print("Best settings:")
    print("Best M (hidden units):", best_M)
    print("Best nHidden (hidden layers):", best_nHidden)
    print("Best learning_rate:", best_lr)
    print("Best l2 regularization:", best_l2)
Esempio n. 13
0
def random_search():  
    # get the data:
    df = pd.read_csv('Spirals_dataset.csv')
    data = df.as_matrix()
    X = data[:,:-1]
    Y = data[:,-1]

    # visualize the data:
    plt.scatter(X[:,0], X[:,1], c=Y)
    plt.title('Spirals')
    plt.axis('equal')
    plt.show()

    # split the data:
    Xtrain, Ytrain = X[:-270, :], Y[:-270]
    Xtest, Ytest = X[-270:, :], Y[-270:]

    # initial hyperparameters:
    M = 20  # number of hidden layer neurons
    num_hl = 2
    log_lr = -4  # learning rate
    log_l2 = -2 # L2-regularization term
    max_tries = 1000
    

    best_validation_score = 0
    best_hl_size = None
    best_lr = None
    best_l2 = None
    t0 = datetime.now()
    # Random Search loop:
    for _ in range(max_tries):
        model = ANN([M]*num_hl)
        model.fit(
            Xtrain,
            Ytrain, 
            learning_rate=10**log_lr, 
            reg=10**log_l2, 
            mu=0.9, 
            epochs=3000, 
            show_fig=False
        )
        validation_score = model.score(Xtest, Ytest)
        train_score = model.score(Xtrain, Ytrain)
        print('\nvalidation set accuracy: %.3f,  training set accuracy: %.3f' % (validation_score, train_score))   
        print('hidden layer size: {}, learning rate: {}, l2: {}'.format([M]*num_hl, 10**log_lr, 10**log_l2))

        if validation_score > best_validation_score:
            best_validation_score = validation_score
            best_hl_size = M
            best_num_hl = num_hl
            best_log_lr = log_lr
            best_log_l2 = log_l2

        # update the hyperparameters:
        log_lr = best_log_lr + np.random.randint(-1, 2)
        log_l2 = best_log_l2 + np.random.randint(-1, 2)
        M = best_hl_size + np.random.randint(-1, 2)*10 # -10, 0, or 10
        M = max(10, M) # in case if M has been updated to 0
        num_hl = best_num_hl + np.random.randint(-1, 2)
        num_hl = max(1, num_hl) # in case if num_hl has been updated to 0


    dt = datetime.now() - t0
    print('\nElapsed time:', dt)
    print('\nBest validation accuracy:', best_validation_score)
    print('\nBest settings:')
    print('hidden layer size:', best_hl_size)
    print('number of hidden layers:', best_num_hl)
    print('learning rate:', 10**best_log_lr)
    print('l2:', 10**best_log_l2)
    print()
Esempio n. 14
0
def random_search():
	max_iter = 30
	X, Y = get_spiral()

	# plt.scatter(X[:,0],X[:,1])
	# plt.show()
	X, Y = shuffle(X,Y)
	Ntrain  =  int(0.7*len(X))
	Xtrain, Ytrain = X[:Ntrain,:], Y[:Ntrain]
	Xtest, Ytest = X[Ntrain:,:], Y[Ntrain:]


	#Starting parameters
	M = 20
	nHidden = 2
	log_lr = -4
	log_l2 = -2

	# best_M = M 
	# best_nHidden = nHidden
	# best_log_lr  = log_lr
	# best_log_l2  = log_l2

	#LOOP thrugh all possible hyperparameter settings
	best_validation_rate = 0 
	best_hls 	= None 
	best_lr 	= None
	best_l2  	=	None 
	for i in xrange(max_iter):
		#PARAMETER SPACE 
		hls = [M]*nHidden
		lr 	= 10**log_lr
		l2  = 10**log_l2

		model = ANN(hls)
		model.fit(Xtrain, Ytrain, learning_rate=lr, reg=12, mu=0.99, epochs=3000, show_fig=False)
		validation_accuracy = model.score(Xtest,Ytest)
		train_accuracy = model.score(Xtrain, Ytrain)
		print(
			"validation_accuracy: %.3f, train_accuracy %.3f, settings: %s, %s, %s" % (validation_accuracy, train_accuracy, hls, lr, l2)
		)
		if validation_accuracy > best_validation_rate:
			best_validation_rate = validation_accuracy
			best_M = M 
			best_nHidden = nHidden
			best_log_lr  = log_lr
			best_log_l2  = log_l2

			best_hls = hls 
			best_lr = lr
			best_l2 = l2


		M 			= max(best_M + np.random.randint(-10, 10)*10, 20)
		nHidden = max(best_nHidden + np.random.randint(-2, 2), 1)
		log_lr  = min(best_log_lr + np.random.randint(-1, 1), -1)
		log_l2  = min(best_log_l2 + np.random.randint(-1, 1), -1)
		print "M", M, "NHIDDEN", nHidden, "LOG-LR", log_lr, "LOG-L2", log_l2
	
	print("Best validation_accuracy", best_validation_rate)						
	print("Best settings:")						
	print("hidden_layer_size:", best_hls)						
	print("learning_rate:",best_lr)						
	print("l2:",best_l2)						
Esempio n. 15
0
def grid_search():
    # get the data:
    df = pd.read_csv('Spirals_dataset.csv')
    data = df.as_matrix()
    X = data[:, :-1]
    Y = data[:, -1]

    # visualize the data:
    plt.scatter(X[:, 0], X[:, 1], c=Y)
    plt.title('Spirals')
    plt.axis('equal')
    plt.show()

    # split the data:
    Xtrain, Ytrain = X[:-270, :], Y[:-270]
    Xtest, Ytest = X[-270:, :], Y[-270:]

    # hyperparameters to be tried:
    M = [[300], [100, 100], [50, 50, 50]]  # number of hidden layer neurons
    learning_rates = [1e-4, 1e-3, 1e-2]  # learning rate
    reg = [0., 1e-1, 1.0]  # L2-regularization term

    best_validation_score = 0
    best_hl_size = None
    best_lr = None
    best_l2 = None
    t0 = datetime.now()
    # Grid Search loops:
    for hl_size in M:
        for lr in learning_rates:
            for l2 in reg:
                model = ANN(hl_size)
                model.fit(Xtrain,
                          Ytrain,
                          learning_rate=lr,
                          reg=l2,
                          mu=0.9,
                          epochs=300,
                          show_fig=False)
                validation_score = model.score(Xtest, Ytest)
                train_score = model.score(Xtrain, Ytrain)
                print(
                    '\nvalidation set accuracy: %.3f,  training set accuracy: %.3f'
                    % (validation_score, train_score))
                print(
                    'hidden layer size: {}, learning rate: {}, l2: {}'.format(
                        hl_size, lr, l2))

                if validation_score > best_validation_score:
                    best_validation_score = validation_score
                    best_hl_size = hl_size
                    best_lr = lr
                    best_l2 = l2
    dt = datetime.now() - t0
    print('\nElapsed time:', dt)
    print('\nBest validation accuracy:', best_validation_score)
    print('\nBest settings:')
    print('Best hidden layer size:', best_hl_size)
    print('Best learning rate:', best_lr)
    print('Best regularization term:', best_l2)
    print()