def calcTestScore(self): legendCounter = 0 learners = [] legend = [] nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='gradient_descent', max_iters=200, bias=True, is_classifier=True, learning_rate=0.001, early_stopping=False, clip_max=1e10, curve=False) learners.append(nn) legend.append('GD') nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='simulated_annealing', max_iters=800, bias=True, is_classifier=True, learning_rate=0.001, early_stopping=False, clip_max=1e10, schedule=mlrose.GeomDecay(), curve=False) learners.append(nn) legend.append('SA') nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='random_hill_climb', max_iters=600, bias=True, is_classifier=True, learning_rate=0.001, early_stopping=False, clip_max=1e10, curve=False) learners.append(nn) legend.append('RHC') nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='genetic_alg', max_iters=200, bias=True, is_classifier=True, learning_rate=0.001, early_stopping=False, clip_max=1e10, pop_size=300, mutation_prob=0.2, curve=False) learners.append(nn) legend.append('GA') for learner in learners: timeStart = time.time() learner.fit(self.dataset1.training_x, self.dataset1.training_y) testingF1 = f1_score(learner.predict(self.dataset1.testing_x), self.dataset1.testing_y, average='weighted') print('Testing F1-Score for' + legend[legendCounter] + ': ' + str(testingF1)) legendCounter += 1
def predict(gradient_descent, cv, iter, restart, X_train, X_test, y_train, y_test): y_train_accuracy = 0 y_test_accuracy = 0 for i in range(restart): # Standardlization scaler = MinMaxScaler() scaler.fit(X_train) X_train = scaler.transform(X_train) X_test = scaler.transform(X_test) # Train Model if algorithm == 'gradient_descent': nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu', algorithm = algorithm, max_iters = iter, bias = True, is_classifier = True, learning_rate = 0.0001, early_stopping = True, max_attempts = 10)#, clip_max = 5) if algorithm == 'random_hill_climb': nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu', algorithm = algorithm, max_iters = iter, bias = True, is_classifier = True, early_stopping = True, max_attempts = 100)#, clip_max = 5) if algorithm == 'simulated_annealing': schedule = mlrose.GeomDecay(init_temp=100, decay=0.5, min_temp=0.1) nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu', algorithm = algorithm, max_iters = iter, bias = True, is_classifier = True, early_stopping = True, max_attempts = 100)#, schedule = schedule)#, clip_max = 5) if algorithm == 'genetic_alg': nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu', algorithm = algorithm, max_iters = iter, bias = True, is_classifier = True, pop_size = 200, mutation_prob = 0.1, early_stopping = True, max_attempts = 1000)#, clip_max = 5) nn_model_cv = nn_model nn_model.fit(X_train, y_train) # Cross Validation if cv == True: pipeline = Pipeline([('transformer', scaler), ('estimator', nn_model_cv)]) cv_scores = cross_val_score(pipeline, X, y, cv=5, scoring='accuracy') # Scores y_train_pred = nn_model.predict(X_train) y_train_accuracy = max(y_train_accuracy, accuracy_score(y_train, y_train_pred)) y_test_pred = nn_model.predict(X_test) y_test_accuracy = max(y_test_accuracy, accuracy_score(y_test, y_test_pred)) print('Training accuracy: ', y_train_accuracy) print('Test accuracy: ', y_test_accuracy) if cv == True: print('Cross validation accuracy: ', cv_scores.mean()) return y_train_accuracy, y_test_accuracy, cv_scores.mean() return y_train_accuracy, y_test_accuracy, y_test_accuracy
def optimize_simulated_annealing(self): nn = mlrose.NeuralNetwork(hidden_nodes=[10], algorithm='simulated_annealing', max_iters=25000, learning_rate=0.5, random_state=7106080, curve=True) hot_training_classes = to_array(self.training_classes, 10) start = time.time() nn.fit(self.training_features, hot_training_classes) elapsed = time.time() - start print('SA took %d ms and %d iterations' % (elapsed * 1000, len(nn.fitness_curve))) plot.clf() plot.plot(nn.fitness_curve) plot.ylabel('fitness') plot.xlabel('iterations') plot.pause(0.001) plot.savefig('sa_fitness_curve_nn') predicted_classes = nn.predict(self.test_features) predicted_classes_not_array = from_array(predicted_classes) print( sklearn.metrics.classification_report(self.test_classes, predicted_classes_not_array)) plot_roc_curves(self.test_classes, nn.predicted_probs) plot.savefig('sa_roc_curve') print( sklearn.metrics.confusion_matrix(self.test_classes, predicted_classes_not_array))
def run_random_hill_experiment_2(x_data, y_data, hidden_nodes): train_scores = [] test_scores = [] train_accs, test_accs = [], [] train_times = [] restarts = [0, 1, 10, 100] FPs, FNs = [], [] random_state = 0 for restart in restarts: print(restart) clf = mlrose.NeuralNetwork( hidden_nodes = hidden_nodes, activation = 'relu', \ algorithm = 'random_hill_climb', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.2, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = None, restarts = restart, curve = True) curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \ return_stratified_kcv_results(clf, x_data, y_data) train_scores.append(train_score) test_scores.append(test_score) train_accs.append(train_acc) test_accs.append(test_acc) train_times.append(train_time) FPs.append(FP) FNs.append(FN) print(FPs) print(FNs) plt.figure(2) plt.plot(restarts, train_scores) plt.plot(restarts, test_scores) plt.plot(restarts, test_accs) plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score']) plt.title('Random Hill Climb performance vs restarts') plt.xlabel('Restarts', fontsize=12) plt.ylabel('Score', fontsize=12) plt.figure(3) plt.plot([i for i in range(0, len(curves))], curves) plt.title('Random Hill Climb fitness at each iteration') plt.xlabel('Iteration', fontsize=12) plt.ylabel('Fitness', fontsize=12) plt.show() plt.figure(4) plt.plot(restarts, train_times) plt.title('Random Hill Climb train time') plt.xlabel('Restarts', fontsize=12) plt.ylabel('Time (s)', fontsize=12) plt.show() plt.figure(5) plt.plot(restarts, FPs) plt.plot(restarts, FNs) plt.legend(['Type 1 Error (FP)', 'Type 2 Error (FN)']) plt.title('Random Hill Climb TYPE 1 (FP) and TYPE 2 (FN) Errors') plt.xlabel('Restarts', fontsize=12) plt.ylabel('Error Rate', fontsize=12) plt.show()
def run_SA_experiment_1(x_data, y_data, hidden_nodes): train_scores = [] test_scores = [] train_accs, test_accs = [], [] learning_rates = [0.001, 1, 2, 5] print('Hidden nodes :', hidden_nodes) for learning_rate in learning_rates: print(learning_rate) clf = mlrose.NeuralNetwork( hidden_nodes = hidden_nodes, activation = 'relu', \ algorithm = 'simulated_annealing', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = learning_rate, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = 30, restarts = 0) curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \ return_stratified_kcv_results(clf, x_data, y_data) print(train_score) train_scores.append(train_score) test_scores.append(test_score) train_accs.append(train_acc) test_accs.append(test_acc) plt.figure(1) plt.plot(learning_rates, train_scores) plt.plot(learning_rates, test_scores) plt.plot(learning_rates, test_accs) plt.legend(['train', 'test', 'acc']) plt.show()
def main(): # Load the Wine dataset full_path = 'data/winequality-white.csv' X, y = load_dataset(full_path) #Split data into training and test sets X_train_scaled, X_test_scaled, y_train_hot, y_test_hot = train_test_split( X, y, test_size=0.2, random_state=RANDOM_STATE) # Initialize neural network object and fit object nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [4], activation = 'relu', \ algorithm = 'random_hill_climb', max_iters = 3000, \ bias = True, is_classifier = True, learning_rate = 0.5, \ early_stopping = True, clip_max = 5, restarts=2, max_attempts = 100, \ random_state = RANDOM_STATE) nn_model1.fit(X_train_scaled, y_train_hot) # Predict labels for train set and assess accuracy y_train_pred = nn_model1.predict(X_train_scaled) y_train_accuracy = accuracy_score(y_train_hot, y_train_pred) print("Train accuracy:", y_train_accuracy) # Predict labels for test set and assess accuracy y_test_pred = nn_model1.predict(X_test_scaled) y_test_accuracy = accuracy_score(y_test_hot, y_test_pred) print("Test accuracy:", y_test_accuracy)
def run_GA_experiment_1(x_data, y_data, hidden_nodes): train_scores = [] test_scores = [] train_accs, test_accs = [], [] train_times = [] iters = [100, 500, 1000, 5000, 10000] FPs, FNs = [], [] for iter in iters: print(iter) clf = mlrose.NeuralNetwork( hidden_nodes = hidden_nodes, activation = 'relu', \ algorithm = 'genetic_alg', max_iters = iter, \ bias = True, is_classifier = True, learning_rate = 0.2, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = 30, pop_size = 100, mutation_prob = 0.1, curve = True) curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \ return_stratified_kcv_results(clf, x_data, y_data, last_curve= True) train_scores.append(train_score) test_scores.append(test_score) train_accs.append(train_acc) test_accs.append(test_acc) train_times.append(train_time) FPs.append(FP) FNs.append(FN) print(FPs) print(FNs) plt.figure(1) plt.plot(iters, train_scores) plt.plot(iters, test_scores) plt.plot(iters, test_accs) plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score']) plt.title('Genetic Algorithm performance vs max iterations') plt.xlabel('Max iterations', fontsize=12) plt.ylabel('Score', fontsize=12) plt.figure(2) plt.plot([i for i in range(0, len(curves))], curves) plt.title('Genetic Algorithm fitness at each iteration') plt.xlabel('Iteration', fontsize=12) plt.ylabel('Fitness', fontsize=12) plt.show() plt.figure(3) plt.plot(iters, train_times) plt.title('Genetic Algorithm train time') plt.xlabel('Iteration', fontsize=12) plt.ylabel('Time (s)', fontsize=12) plt.show() plt.figure(4) plt.plot(iters, FPs) plt.plot(iters, FNs) plt.legend(['Type 1 Error (FP)', 'Type 2 Error (FN)']) plt.title('Genetic Algorithm TYPE 1 (FP) and TYPE 2 (FN) Error Rates') plt.xlabel('Max iterations', fontsize=12) plt.ylabel('Error Rate', fontsize=12) plt.show()
def main(): # Load the Wine dataset full_path = 'data/winequality-white.csv' X, y = load_dataset(full_path) #Split data into training and test sets X_train_scaled, X_test_scaled, y_train_hot, y_test_hot = train_test_split( X, y, test_size=0.2, random_state=RANDOM_STATE) # Initialize neural network object and fit object t_bef = time.time() nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [2], activation = 'tanh', \ algorithm = 'genetic_alg', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.0001, \ early_stopping = True, clip_max = 5, pop_size=900, mutation_prob=0.1, max_attempts = 100, \ random_state = RANDOM_STATE) nn_model1.fit(X_train_scaled, y_train_hot) t_aft = time.time() curve = nn_model1.fitness_curve # Predict labels for train set and assess accuracy y_train_pred = nn_model1.predict(X_train_scaled) y_train_accuracy = accuracy_score(y_train_hot, y_train_pred) print("Train accuracy:", y_train_accuracy) # Predict labels for test set and assess accuracy y_test_pred = nn_model1.predict(X_test_scaled) y_test_accuracy = accuracy_score(y_test_hot, y_test_pred) print("Test accuracy:", y_test_accuracy)
def run_random_hill_experiment_1(x_data, y_data, hidden_nodes): train_scores = [] test_scores = [] train_accs, test_accs = [], [] learning_rates = [0.001, 0.01, 0.1, 0.2, 0.5, 1] for learning_rate in learning_rates: print(learning_rate) clf = mlrose.NeuralNetwork( hidden_nodes = hidden_nodes, activation = 'relu', \ algorithm = 'random_hill_climb', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = learning_rate, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = 30, restarts = 0) curve, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \ return_stratified_kcv_results(clf, x_data, y_data) train_scores.append(train_score) test_scores.append(test_score) train_accs.append(train_acc) test_accs.append(test_acc) plt.figure(1) plt.plot(learning_rates, train_scores) plt.plot(learning_rates, test_scores) plt.plot(learning_rates, test_accs) plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score']) plt.title('Random Hill Climb performance vs learning rates') plt.xlabel('Learning rate', fontsize=12) plt.ylabel('Score', fontsize=12) plt.show()
def MLP_rhc(X, y, X_test, y_test): print('Running Randomized Hill Climbing') # Perform Random Hill Climbing train_errors = [] test_errors = [] for i in range(0, 6): print(i) clf_rhc = mlrose.NeuralNetwork(hidden_nodes=[8, 8], activation='tanh', is_classifier=True, algorithm='random_hill_climb', max_iters=500, bias=True, learning_rate=0.01, early_stopping=False, clip_max=100.0, restarts=i, max_attempts=10, curve=True, random_state=1) clf_rhc = clf_rhc.fit(X, y) train_errors.append(clf_rhc.loss) test_errors.append(clf_rhc.score(X_test, y_test)) clf_rhc = mlrose.NeuralNetwork(hidden_nodes=[8, 8], activation='tanh', is_classifier=True, algorithm='random_hill_climb', max_iters=500, bias=True, learning_rate=0.01, early_stopping=False, clip_max=100.0, restarts=1, max_attempts=10, curve=True, random_state=1) clf_rhc = clf_rhc.fit(X, y) pred_score_rhc = [clf_rhc.score(X_test, y_test)] PlotData(list(range(len(clf_rhc.fitness_curve))), clf_rhc.fitness_curve, list(range(len(pred_score_rhc))), pred_score_rhc, 'Error vs Iterations') return clf_rhc, train_errors, test_errors
def nn_ga(train_x, train_y, test_x, test_y): activations = ['identity', 'relu', 'sigmoid', 'tanh'] markers = ['o', 'x', '^', 's'] colors = ['g', 'b', 'r', 'c', 'm', 'k'] pop_sizes = [50, 200] mutation_probs = [0.1, 0.5] legend_items = [ mlines.Line2D([0], [0], color='g', lw=2), mlines.Line2D([0], [0], color='b', lw=2), mlines.Line2D([0], [0], color='r', lw=2), mlines.Line2D([0], [0], color='c', lw=2), mlines.Line2D([0], [0], color='m', lw=2), mlines.Line2D([0], [0], color='k', lw=2) ] legend_labels = [ 'pop 50, prob 0.1', 'pop 200, prob 0.1', 'pop 50, prob 0.5', 'pop 200, prob 0.5' ] counter = 0 for index, mutation_prob in enumerate(mutation_probs): for jindex, pop_size in enumerate(pop_sizes): nn_model = mlrose.NeuralNetwork(hidden_nodes=[8, 6], activation='tanh', algorithm='genetic_alg', pop_size=pop_size, mutation_prob=mutation_prob, max_iters=1000, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=1000, max_attempts=100, random_state=3, curve=True) fitted_model = nn_model.fit(train_x, train_y) plt.plot(fitted_model.fitness_curve, c=colors[counter]) counter += 1 plt.title("NN Training: GA") plt.xlabel("No. of Iterations") plt.ylabel("Fitness") plt.legend(legend_items, legend_labels, loc="lower right") plt.tight_layout() plt.savefig("nn_ga") plt.cla()
def run_rhc_nn(name, x_train, x_test, y_train, y_test): print ("Working on RHC NN...") report_name = "reports/{}_nn_rhc_output.txt".format(name) sys.stdout = open(report_name, "w") rhc_nn = mlrose.NeuralNetwork(hidden_nodes = [11], algorithm = 'random_hill_climb', max_iters = 1000) run_optimized(rhc_nn, x_train, y_train, x_test, y_test) sys.stdout = sys.__stdout__
def rhc(self, max_iter=10000, restarts=0): learner = mlrose.NeuralNetwork(hidden_nodes=[2], activation='relu', algorithm='random_hill_climb', max_iters=max_iter, bias=True, learning_rate=3e-03, early_stopping=True, max_attempts=1000, restarts=restarts) return learner
def run_sa_nn(name, x_train, x_test, y_train, y_test): print ("Working on SA NN...") report_name = "reports/{}_nn_sa_output.txt".format(name) sys.stdout = open(report_name, "w") sa_nn = mlrose.NeuralNetwork(hidden_nodes = [11], algorithm = 'simulated_annealing', max_iters = 1000) run_optimized(sa_nn, x_train, y_train, x_test, y_test) sys.stdout = sys.__stdout__
def model_complexity_exp_restart_rhc(self): #TODO should we create a new learner object?? scoring = 'accuracy' nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \ algorithm = 'random_hill_climb', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.0001, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = 3, restarts = 0) expHelper = ExperimentHelper(self.splitter, nn_model1, 'RHC') param_range = [1, 10, 50, 100, 200, 300, 400, 500] expHelper.model_complexity_exp('restarts', param_range)
def model_complexity_exp(self): scoring = 'accuracy' nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \ algorithm = 'random_hill_climb', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.0001, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = 3) expHelper = ExperimentHelper(self.splitter, nn_model1, 'RHC') param_range = np.array([0.0001, 0.001, 0.002, 0.003, 0.005, 0.008]) #param_range = np.array([100, 200,300,400, 500]) expHelper.model_complexity_exp('learning_rate', param_range)
def model_complexity_exp_max_attempt_ga(self): #TODO should we create a new learner object?? scoring = 'accuracy' nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \ algorithm = 'genetic_alg', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.0001, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ pop_size= 2000, random_state = 3) expHelper = ExperimentHelper(self.splitter, nn_model1, 'GA') param_range = [20, 50, 100, 200, 300] expHelper.model_complexity_exp('max_attempts', param_range)
def run(self, n=1): """ n : the number of runs to do returns best_fitnesses (list), learning_curves (list) """ log_loss_curves = [] train_acc = [] test_acc = [] train_f1 = [] test_f1 = [] fitted_weights = [] loss = [] for i in np.arange(n): X_train, X_test, y_train, y_test = data_helper.train_test_split( self.X, self.y, seed=None, test_size=self.test_size) model = mlrose.NeuralNetwork(hidden_nodes=self.hidden_nodes, activation=self.activation, algorithm=self.algorithm, max_iters=self.max_iters, bias=True, is_classifier=True, learning_rate=self.learning_rate, early_stopping=False, clip_max=self.clip_max, restarts=self.restarts, schedule=self.schedule, pop_size=self.pop_size, mutation_prob=self.mutation_prob, max_attempts=self.max_attempts, curve=True) model.fit(X_train, y_train) y_train_pred = model.predict(X_train) y_train_accuracy = metrics.accuracy_score(y_train, y_train_pred) train_acc.append(y_train_accuracy) train_f1.append( metrics.f1_score(y_train, y_train_pred, average='weighted')) y_test_pred = model.predict(X_test) y_test_accuracy = metrics.accuracy_score(y_test, y_test_pred) test_acc.append(y_test_accuracy) test_f1.append( metrics.f1_score(y_test, y_test_pred, average='weighted')) log_loss_curve = np.array(model.fitness_curve) * -1 log_loss_curves.append(log_loss_curve) # print(model.fitted_weights.shape) # fitted_weights.append(model.fitted_weights) loss.append(model.loss) # print("train ", train_acc) # print("test", test_acc) # print(np.array(log_loss_curves).shape) return train_acc, train_f1, test_acc, test_f1, np.array( log_loss_curves), loss
def create_nn_schedule(decay): if decay == 'Geom': schedule = mlrose.GeomDecay() if decay == 'Exp': schedule = mlrose.ExpDecay() if decay == 'Arith': schedule = mlrose.ArithDecay() return mlrose.NeuralNetwork(hidden_nodes=[10], algorithm='simulated_annealing', max_iters=1000, schedule=schedule, random_state=7106080)
def ga(self, max_iter=10000, mutation_prob=0.1, pop_size=200): learner = mlrose.NeuralNetwork(hidden_nodes=[2], activation='relu', algorithm='genetic_alg', max_iters=max_iter, bias=True, learning_rate=3e-03, early_stopping=True, max_attempts=100, mutation_prob=mutation_prob, pop_size=pop_size) return learner
def model_complexity_exp_epoch_no_stop_sa(self): #TODO should we create a new learner object?? scoring = 'accuracy' nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \ algorithm = 'simulated_annealing', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.0001, \ early_stopping = False, clip_max = 5, max_attempts = 100, \ random_state = 3) expHelper = ExperimentHelper(self.splitter, nn_model1, 'SA') param_range = [1, 10, 50, 100, 200, 500] expHelper.model_complexity_exp('max_iters', param_range)
def sa(self, max_iter=10000, schedule=None): if schedule is None: schedule = mlrose.ExpDecay() learner = mlrose.NeuralNetwork(hidden_nodes=[2], activation='relu', algorithm='simulated_annealing', max_iters=max_iter, bias=True, learning_rate=3e-03, early_stopping=True, max_attempts=1000, schedule=schedule) return learner
def nn_sa_temps(train_x, train_y, test_x, test_y): min_temps = [1, 5] init_temps = [100, 500] markers = ['o', 'x', '^', 's'] colors = ['g', 'b', 'r', 'c', 'y', 'm'] counter = 0 for index, min_temp in enumerate(min_temps): for jindex, init_temp in enumerate(init_temps): schedule = mlrose.ExpDecay(init_temp=init_temp, exp_const=0.001, min_temp=min_temp) nn_model = mlrose.NeuralNetwork(hidden_nodes=[8, 6], activation='tanh', algorithm='simulated_annealing', schedule=schedule, max_iters=1000, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=1000, max_attempts=100, random_state=3, curve=True) fitted_model = nn_model.fit(train_x, train_y) label = "min_temp: " + str(min_temp) + ", init_temp: " + str( init_temp) plt.plot(fitted_model.fitness_curve, c=colors[counter], label=label) counter += 1 plt.xlabel("No. of Iterations") plt.ylabel("Fitness") plt.title("NN Training: SA") plt.legend(loc="lower right") plt.tight_layout() plt.savefig("nn_sa_temps") plt.cla()
def model_complexity_exp_max_attempts_sa(self): #TODO should we create a new learner object?? scoring = 'accuracy' decay = mlrose.decay.ExpDecay() nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \ algorithm = 'simulated_annealing', max_iters = 1000, \ bias = True, is_classifier = True, learning_rate = 0.0001, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ schedule = decay, random_state = 3) expHelper = ExperimentHelper(self.splitter, nn_model1, 'SA') param_range = [20, 50, 100, 200, 300] expHelper.model_complexity_exp('max_attempts', param_range) print('completed max attempt sa nn')
def genetic(iterations): nn_model_genetic = mlrose.NeuralNetwork(hidden_nodes=[4], activation='relu', algorithm='genetic_alg', max_iters=iterations, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=5, max_attempts=100, random_state=3) nn_model_genetic.fit(X_train, y_train) y_test_pred = nn_model_genetic.predict(X_test_scaled) y_test_accuracy = accuracy_score(y_test, y_test_pred) return (y_test_pred, y_test_accuracy)
def simulated_annealing(iterations): nn_model = mlrose.NeuralNetwork(hidden_nodes=[4], activation='relu', algorithm='simulated_annealing', max_iters=iterations, is_classifier=True, learning_rate=0.00001, early_stopping=True, clip_max=5, random_state=156) nn_model.fit(X_train, y_train) y_test_pred = nn_model.predict(X_test) y_test_accuracy2 = accuracy_score(y_test, y_test_pred) return (y_test_pred, y_test_accuracy2)
def run_GA_experiment_3(x_data, y_data, hidden_nodes): train_scores = [] test_scores = [] train_accs, test_accs = [], [] train_times = [] pop_sizes = [50, 2000] print('Hidden nodes :', hidden_nodes) for pop_size in pop_sizes: clf = mlrose.NeuralNetwork( hidden_nodes = hidden_nodes, activation = 'relu', \ algorithm = 'genetic_alg', max_iters = 200, \ bias = True, is_classifier = True, learning_rate = 0.2, \ early_stopping = True, clip_max = 5, max_attempts = 100, \ random_state = 30, pop_size = pop_size, mutation_prob = 0.1, curve = True) curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \ return_stratified_kcv_results(clf, x_data, y_data, last_curve=True) train_scores.append(train_score) test_scores.append(test_score) train_accs.append(train_acc) test_accs.append(test_acc) train_times.append(train_time) plt.figure(7) plt.plot(pop_sizes, train_scores) plt.plot(pop_sizes, test_scores) plt.plot(pop_sizes, test_accs) plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score']) plt.title('Genetic Algorithm performance vs population size') plt.xlabel('Population sizes', fontsize=12) plt.ylabel('Score', fontsize=12) plt.figure(8) plt.plot([i for i in range(0, len(curves))], curves) plt.title('Genetic Algorithm fitness at each iteration') plt.xlabel('Iteration', fontsize=12) plt.ylabel('Fitness', fontsize=12) plt.show() plt.figure(9) plt.plot(pop_sizes, train_times) plt.title('Genetic Algorithm train time') plt.xlabel('population sizes', fontsize=12) plt.ylabel('Time (s)', fontsize=12) plt.show()
def train_and_validate(X, y, X_t, y_t): model = mlrose.NeuralNetwork(hidden_nodes = [50, 20, 8, 2], activation = "relu", \ bias = True, is_classifier = True, early_stopping = True, \ algorithm="gradient_descent", \ max_iters=1000, max_attempts = 100, learning_rate = 0.001) start_train = time.time() model.fit(X, y) print("training time: " + str(time.time() - start_train)) print("loss: " + str(model.loss)) pred = model.predict(X) print("training data accuracy: " + str(round(accuracy_score(y, pred) * 100, 2)) + "%") test_pred = model.predict(X_t) print("testing data accuracy: " + str(round(accuracy_score(y_t, test_pred) * 100, 2)) + "%")
def learn_rate(lr): test_accuracy = [] train_accuracy = [] learning_rate = [] for c in lr: #schedule=mlrose.GeomDecay(init_temp=c) nn_model1 = mlrose.NeuralNetwork(hidden_nodes=[4, 4, 4], activation='relu', algorithm='genetic_alg', max_iters=1000, mutation_prob=0.4, bias=True, learning_rate=0.01, pop_size=200, is_classifier=True, early_stopping=True, clip_max=5, max_attempts=100) nn_model1.fit(X_train, y_train) y_train_pred = nn_model1.predict(X_train) y_test_pred = nn_model1.predict(X_test) y_train_accuracy = accuracy_score(y_train, y_train_pred) y_test_accuracy = accuracy_score(y_test, y_test_pred) test_accuracy.append(y_test_accuracy) train_accuracy.append(y_train_accuracy) learning_rate.append(c) print(test_accuracy) #print(nn_model1.fitted_weights) train_accuracy = np.asarray(train_accuracy) test_accuracy = np.asarray(test_accuracy) learning_rate = np.asarray(learning_rate) line1, = plt.plot(learning_rate, train_accuracy, color='r', label='train_accuracy_uniform') line2, = plt.plot(learning_rate, test_accuracy, color='b', label='test_accuracy_uniform') plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)}) plt.ylabel('Accuracy_score') plt.xlabel('mutation probability') return None
def model(algorithm, x_train, y_train, x_test, iterations=100, activation='relu'): nn = mlrose.NeuralNetwork( hidden_nodes=[4], activation=activation, algorithm=algorithm, max_iters=iterations, is_classifier=True, learning_rate=0.0001, # random_state = 2 ) nn.output_activation = mlrose.activation.sigmoid nn.fit(x_train, y_train) y_pred = nn.predict(x_test) return y_pred