def neural_network_tune_sa(): features, labels = load_data('dataset1', 'train') x_train, x_test, y_train, y_test = train_test_split( features, labels, test_size=0.2, random_state=RANDOM_STATE) decay = [ mlr.GeomDecay(init_temp=1.0, decay=0.99, min_temp=0.001), mlr.GeomDecay(init_temp=1.0, decay=0.8, min_temp=0.001), mlr.GeomDecay(init_temp=1.0, decay=0.6, min_temp=0.001), mlr.GeomDecay(init_temp=1.0, decay=0.4, min_temp=0.001), mlr.GeomDecay(init_temp=1.0, decay=0.2, min_temp=0.001) ] for i in decay: algorithm = 'simulated_annealing' sa = mlr.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu', \ algorithm = algorithm, max_iters=5000, schedule=i, \ bias = True, is_classifier = True, learning_rate = 0.1, \ early_stopping = True, clip_max = 1, max_attempts = 100, \ random_state = RANDOM_STATE, curve=True) print('Simulated Annealing Decay Rate=', i) start_time = time.time() sa.fit(x_train, y_train) end_time = time.time() total_time = end_time - start_time print("Fit Time", total_time) y_train_pred = sa.predict(x_train) y_train_accuracy = accuracy_score(y_train, y_train_pred) print('Train Score', y_train_accuracy) y_test_pred = sa.predict(x_test) y_test_accuracy = accuracy_score(y_test, y_test_pred) print('Validation Score:', y_test_accuracy)
def NN_SA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) # SA HPs nodes = [128, 128, 128, 128] act = 'relu' seed = 1 sa_algo = 'simulated_annealing' sa_lr = 10 sa_iter = 10000 sa_temp = 10000 sa_decay = 0.92 sa_ma = 50 sa_clip = 10 temperature = [0.1, 1, 10, 100, 1000, 10000] plt.figure() for t in temperature: print('temperature', t) sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo, max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma, schedule=mlrose.GeomDecay(init_temp=t, decay=sa_decay)) sa_nn_model.fit(X_train, y_train) plt.plot(sa_nn_model.fitness_curve, label='temp =' + str(t)) plt.title("NN SA - Temperature") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.savefig("Images\\NN - SA - Temperature") plt.xscale('log') plt.savefig("Images\\NN - SA - Temperature - log") plt.show() decay_rates = [0.1, 0.2, 0.3, 0.4, 0.5, 0.8, 0.92] plt.figure() for dr in decay_rates: print('decay', dr) sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo, max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma, schedule=mlrose.GeomDecay(init_temp=sa_temp, decay=dr)) sa_nn_model.fit(X_train, y_train) plt.plot(sa_nn_model.fitness_curve, label='decay rate =' + str(dr)) plt.title("NN SA - Decay Rate") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.savefig("Images\\NN - SA - Decay Rate") plt.xscale('log') plt.savefig("Images\\NN - SA - Decay Rate - log") plt.show()
def sa(self): iteration = self.noOfiteration problem_size_space = self.problem_size step = problem_size_space // 2 # temperature_list=[1, 10, 50, 100, 250, 500, 1000, 2500, 5000, 10000] temperature_list = [1, 10, 50] # sa_params = { 'hidden_nodes': [(3,), (4,), (5,), (5, 5)], sa_params = { 'hidden_nodes': [(3, ), (5, ), (5, 5), (10, 10, 10), (5, 5, 5)], #(10, 10), (5, 5, 5), (10, 10, 10), (20, 20, 20)] 'max_iters': [x for x in range(0, iteration + 1, 1000)], 'schedule': [ mlrose.GeomDecay(init_temp=init_temp) for init_temp in temperature_list ], 'learning_rate': [0.001, 0.01, 0.1], 'activation': ['tanh', 'relu', 'sigmoid'] } sa_model = mlrose.NeuralNetwork(random_state=1, algorithm='simulated_annealing', bias=False, is_classifier=True, learning_rate=0.001, early_stopping=True, clip_max=5, max_attempts=5000, curve=True) return sa_model, sa_params
def n_queens_sa(nq_problem, initial_state, max_iters=np.inf, num_runs=20, verbose=False): hp_name = 'schedule' hp_values = [mlrose.ArithDecay(), mlrose.GeomDecay(), mlrose.ExpDecay()] hp_values_strings = [ val.get_info__()['schedule_type'] for val in hp_values ] # run for each hp value and append results to list fitness_dfs = [] runs = np.arange(num_runs) for hp_value, hp_value_string in zip(hp_values, hp_values_strings): schedule = hp_value # set varied HP at beginning of loop run_times = np.zeros(num_runs) fitness_data = pd.DataFrame() for run in runs: run_t0 = time() best_state, best_fitness, fitness_curve = mlrose.simulated_annealing( problem=nq_problem, schedule=schedule, max_attempts=10, max_iters=max_iters, curve=True, ) run_time = time() - run_t0 run_times[run] = run_time fitness_data = pd.concat( [fitness_data, pd.DataFrame(fitness_curve)], axis=1, sort=False) fitness_data.columns = runs fitness_data = fitness_data.fillna(method='ffill') fitness_dfs.append(fitness_data) # calculate and print avg time per run avg_run_time = np.average(run_times) print("N-Queens - SA avg run time,", hp_value_string, hp_name, ":", avg_run_time) # generate plots plot_title = "N-Queens SA: fitness vs. iterations" plotting.plot_fitness_curves( fitness_dfs=fitness_dfs, hp_values=hp_values_strings, hp_name=hp_name, title=plot_title, ) plt.savefig('graphs/n_queens_sa_fitness.png') plt.clf() return fitness_dfs
def SA_best_params(problem, params_dict, inverse_fitness=False): init_temp_list = params_dict['init_temp'] decay_list = params_dict['decay'] max_attempts_list = params_dict['max_attempts'] max_iters_list = params_dict['max_iters'] max_iters_list = [max_iters_list[-1]] rand_list = params_dict['random_state'] best_params_df = pd.DataFrame({'max_iter':[0], 'avg_fit':[999], 'max_fit':[999], 'min_fit':[999], 'avg_time':[0]\ , 'init_temp':[0],'decay':[0], 'max_attempts':[0]}) for t in init_temp_list: for d in decay_list: for ma in max_attempts_list: temp_schedule = mlrose.GeomDecay(init_temp=t, decay=d) temp_results_df = fitness_by_iter('SA', problem, max_iters_list, rand_list, schedule=temp_schedule\ , init_state=None, max_attempts=ma, curve=False) temp_results_df['init_temp'] = t temp_results_df['decay'] = d temp_results_df['max_attempts'] = ma best_params_df = pd.concat([best_params_df, temp_results_df]).copy() best_params_df = best_params_df[(best_params_df['max_iter'] > 0)].copy() if inverse_fitness == True: best_params_df['avg_fit'] = best_params_df['avg_fit'].apply( lambda x: 1 / x) best_params_df['max_fit'] = best_params_df['max_fit'].apply( lambda x: 1 / x) best_params_df.sort_values('avg_fit', ascending=False) return best_params_df
def call_mlrose_curve(algorith_keyword, problem, pop_size=200, mutation_prob=0.1, max_attempts=10, max_iters=np.inf, curve=False\ , random_state=None, schedule=mlrose.GeomDecay(), init_state=None, restarts=0, keep_pct=0.2, fast_mimic=True): if algorith_keyword == 'RHC': best_state, best_fitness, curve_output = mlrose.random_hill_climb(problem, max_attempts=max_attempts, max_iters=max_iters\ , restarts=restarts, init_state=init_state, curve=curve, random_state=random_state) elif algorith_keyword == 'GA': best_state, best_fitness, curve_output = mlrose.genetic_alg(problem, pop_size=pop_size, mutation_prob=mutation_prob\ , max_attempts=max_attempts, max_iters=max_iters, curve=curve, random_state=random_state) elif algorith_keyword == 'SA': best_state, best_fitness, curve_output = mlrose.simulated_annealing(problem, schedule=schedule, max_attempts=max_attempts\ ,max_iters=max_iters, init_state=init_state, curve=curve, random_state=random_state) elif algorith_keyword == 'MIMIC': print("problem: ", problem, "\npop_size: ", pop_size, "\n", "keep_pct: ", keep_pct) print("max_attempts: ", max_attempts, "\nmax_iters: ", max_iters, "\nrandom_state: ", random_state, "\nfast_mimic: ", fast_mimic) best_state, best_fitness, curve_output = mlrose.mimic(problem, pop_size=pop_size, keep_pct=keep_pct\ , max_attempts=max_attempts, max_iters=max_iters, curve=curve, random_state=random_state) print("best_fitness: ", best_fitness) else: print( "\n\nIncorrect 'algorithm_keyword'. Please check the input to the 'call_mlrose' function.\n\n" ) best_state, best_fitness, curve_output = 'incorrect key word', 'incorrect key word', 'incorrect key word' return best_state, best_fitness, curve_output
def simulated_annealing(problem_fit, vectorLength, data_dir, iterlist): directory = "./" + data_dir + "/curves/" if not os.path.exists(directory): os.makedirs(directory) path1 = './' + data_dir path2 = "./" + data_dir + "/curves/" beststate = [] bestfit = [] curve = [] time = [] iterlistn = [] CEl = [] for iters in iterlist: for CE in [0.20, 0.40, 0.60, 0.80, 1.0]: CEl.append(CE) start = clock() best_state, best_fitness, train_curve = mlrose.simulated_annealing(problem_fit,\ max_iters=int(iters),\ curve=True, \ schedule=mlrose.GeomDecay(init_temp=CE), random_state=randomSeed) end = clock() time.append(end - start) beststate.append(best_state) bestfit.append(best_fitness) curve.append(train_curve) iterlistn.append(int(iters)) if (verbose == True): print(CE) print(int(iters)) print(best_state) print(best_fitness) ffsa = pd.DataFrame({ 'Best Fitness': bestfit, 'Iterations': iterlistn, 'Time': time, 'CE': CEl }) beststatedf = pd.DataFrame(0.0, index=range(1, vectorLength + 1), columns=range(len(beststate))) for i in range(len(curve)): pd.DataFrame(curve[i]).to_csv( os.path.join(path2, 'sacurve_{}_{}.csv'.format(iterlistn[i], CEl[i]))) for i in range(1, len(beststate) + 1): beststatedf.loc[:, i] = beststate[i - 1] ffsa.to_csv(os.path.join(path1, 'sa.csv')) beststatedf.to_csv(os.path.join(path1, 'sastates.csv'))
def mlp_sa(max_iters): return mlrose_hiive.NeuralNetwork( hidden_nodes=[10, 10, 10], algorithm='simulated_annealing', activation='tanh', learning_rate=0.1, # 0.1, 0.01, 0.001, 0.0001, 0.000001 max_iters=max_iters, early_stopping=True, schedule=mlrose_hiive.GeomDecay(), # ArithDecay, ExpDecay, GeomDecay random_state=SEED, max_attempts=100 # 10, 100, 500 )
def get_sim_ann(problem, schedule=mlrose.GeomDecay()): best_state, best_fitness, fitness_curve = mlrose.simulated_annealing( problem, schedule=schedule, max_attempts=100, max_iters=np.inf, init_state=None, curve=True, random_state=2) return best_state, best_fitness, fitness_curve
def fitness_by_iter(keyword, problem, max_iters_list, rand_list, pop_size=200, mutation_prob=0.1, max_attempts=10, curve=False\ , schedule=mlrose.GeomDecay(), init_state=None, restarts=0, keep_pct=0.2, fast_mimic=True): avg_fit_list = [] max_fit_list = [] min_fit_list = [] avg_time_list = [] curve_output_list = [] for m in max_iters_list: temp_fit_list = [] temp_time_list = [] temp_curve_list = [] for r in rand_list: if curve == False: start_time_fit = time.perf_counter() best_state, best_fitness = call_mlrose(keyword, problem, random_state=r, max_iters=m, max_attempts=max_attempts\ , pop_size=pop_size, mutation_prob=mutation_prob, curve=curve, schedule=schedule\ , init_state=init_state, restarts=restarts, keep_pct=keep_pct, fast_mimic=fast_mimic) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit temp_fit_list.append(best_fitness) temp_time_list.append(time_used) else: start_time_fit = time.perf_counter() best_state, best_fitness, curve_output = call_mlrose(keyword, problem, random_state=r, max_iters=m\ , max_attempts=max_attempts, pop_size=pop_size, mutation_prob=mutation_prob, curve=True, schedule=schedule\ , init_state=init_state, restarts=restarts, keep_pct=keep_pct, fast_mimic=fast_mimic) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit temp_fit_list.append(best_fitness) temp_time_list.append(time_used) curve_output2 = np.zeros(m) temp_len = len(curve_output) curve_output2[0:temp_len] = curve_output temp_curve_list.append(curve_output2) avg_fit = np.average(np.array(temp_fit_list)) max_fit = np.max(np.array(temp_fit_list)) min_fit = np.min(np.array(temp_fit_list)) avg_time = np.average(np.array(temp_time_list)) if curve == True: avg_curve_pre = np.vstack(temp_curve_list) avg_curve = np.mean(avg_curve_pre, axis=0) curve_output_list.append(avg_curve) avg_fit_list.append(avg_fit) max_fit_list.append(max_fit) min_fit_list.append(min_fit) avg_time_list.append(avg_time) results_df = pd.DataFrame({'max_iter':max_iters_list, 'avg_fit':avg_fit_list, 'max_fit':max_fit_list\ ,'min_fit':min_fit_list, "avg_time":avg_time_list}) if curve == False: return results_df else: return results_df, curve_output_list
def sa_optimization(size): algo = 'sa' problem = get_problem(size) # Gridsearch params max_iters = 500 schedule_values = [mlrose_hiive.GeomDecay(), mlrose_hiive.ArithDecay(), mlrose_hiive.algorithms.decay.ExpDecay()] max_attempts_values = [10, 50, 100, 200] n_runs = len(schedule_values) * len(max_attempts_values) # Best vals schedule, max_attempts = None, None fitness, curves, n_invocations, time = float('-inf'), [], 0, 0 # Gridsearch global eval_count run_counter = 0 for run_schedule in schedule_values: for run_max_attempts in max_attempts_values: # Print status run_counter += 1 print(f'RUN {run_counter} of {n_runs} [schedule: {run_schedule.__class__.__name__}] [max_attempts: {run_max_attempts}]') # Run problem eval_count = 0 start = timer() run_state, run_fitness, run_curves = mlrose_hiive.simulated_annealing(problem, schedule=run_schedule, max_attempts=run_max_attempts, max_iters=max_iters, random_state=42, curve=True) end = timer() # Save curves and params if run_fitness > fitness: schedule = run_schedule.__class__.__name__ max_attempts = run_max_attempts fitness = run_fitness curves = run_curves n_invocations = eval_count time = end - start df = pandas.DataFrame(curves, columns=['fitness']) df['schedule'] = schedule df['max_attempts'] = max_attempts df['max_iters'] = max_iters df['n_invocations'] = n_invocations df['time'] = time df.to_csv(f'{STATS_FOLDER}/{algo}_{size}_stats.csv', index=False) print(f'{algo}_{size} run.')
def sa_nn(data): results = [] for decay in [0.95, 0.975, 0.99, 0.995]: nn = mlrose.NeuralNetwork(hidden_nodes=[8, 8], activation='relu', algorithm='simulated_annealing', max_iters=10000, learning_rate=0.1, early_stopping=True, max_attempts=500, schedule=mlrose.GeomDecay(decay=decay), clip_max=5, random_state=0, curve=True) results.append(run_nn(*data, nn, decay=decay)) return results
def test_param(param_name, param_range, problem, algorithm, args, plot=False, alg_name="", prob_name=""): args['problem'] = problem fits = [] best_fit = 0 best_arg_val = 0 if param_name == 'decay': #this is SA, we need to define schedule for p in param_range: print(p) args['schedule'] = mlrose.GeomDecay(init_temp=10, decay=p, min_temp=0.001) state, fitness, curve = algorithm(**args) fits.append(fitness) if fitness > best_fit: best_fit = fitness best_arg_val = p else: for p in param_range: args[param_name] = p state, fitness, curve = algorithm(**args) fits.append(fitness) if fitness > best_fit: best_fit = fitness best_arg_val = p print("best value for ", param_name, ": ", best_arg_val) print("best fit: ", best_fit) if plot: plt.plot(param_range, fits, label=alg_name) plt.xlabel(param_name) plt.ylabel('fit') plt.legend() title = "" + alg_name + " " + param_name + " fits " + prob_name plt.title(title) plt.grid(True, linestyle='-.') plt.show() return best_arg_val, best_fit
def plot_SAdecay(problem, problem_fit, max_attempts, init_temp, maxIter, seed, min=False): decay_r = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95] plt.figure() for d in decay_r: SAschedule = mlrose.GeomDecay(init_temp=init_temp, decay=d, min_temp=0.01) best_state, best_fitness, safitness_curve = mlrose.simulated_annealing(problem_fit, schedule=SAschedule, curve=True, max_attempts=max_attempts, random_state=seed, max_iters=maxIter) if min: safitness_curve = np.array(safitness_curve) * -1 plt.plot(safitness_curve, label='decay rate = ' + str(d)) plt.title(problem + " - SA - Decay Rates") plt.xlabel('Iterations') plt.ylabel('Fitness') plt.grid(True) plt.legend() plt.savefig("Images\\" + problem + " - SA - Decay Rates") plt.show()
def plot_SATemps(problem, problem_fit, max_attempts, decay, maxIter, seed, min=False): temps = [10000000, 1000000, 100000, 10000, 1000, 100, 10, 1, 0.1] plt.figure() for t in temps: SAschedule = mlrose.GeomDecay(init_temp=t, decay=decay, min_temp=0.01) best_state, best_fitness, safitness_curve = mlrose.simulated_annealing(problem_fit, schedule=SAschedule, curve=True, max_attempts=max_attempts, random_state=seed, max_iters=maxIter) if min: safitness_curve = np.array(safitness_curve) * -1 plt.plot(safitness_curve, label='Temperature = ' + str(t)) plt.title(problem + " - SA - Initial Temperature") plt.xlabel('Iterations') plt.ylabel('Fitness') plt.grid(True) plt.legend() plt.savefig("Images\\" + problem + " - SA - Initial Temperature") plt.show()
def runSA(self): default = { 'problem': self.problem, 'schedule': self.schedule, 'max_attempts': 10, 'max_iters': 1000, 'init_state': self.init_state, 'curve': True, 'random_state': 1 } maxAttempts = [5, 10, 20] schedules = [mlrose.GeomDecay(), mlrose.ExpDecay(), mlrose.ArithDecay()] bestFitness = None (bestState, bestCurve, bestParams) = None, None, None for i in maxAttempts: for j in schedules: params = _.assign( {}, default, {'max_attempts': i, 'schedule': j}) scores = [] for r in range(5): randomSeed = np.random.randint(0, 1000) params = _.assign( {}, params, {'random_state': randomSeed}) state, fitness, curve = self._run( mlrose.simulated_annealing, name='%s' % i, **params) scores.append(fitness) avgFitness = np.mean(scores) if bestFitness == None or (self.isMaximize and avgFitness > bestFitness) or (not self.isMaximize and avgFitness < bestFitness): bestFitness = avgFitness (bestState, bestCurve, bestParams) = state, curve, params # if fitness == 0: # break print('SA - Params: %s' % bestParams) log.info('\tSA - Best fitness found: %s\n\t\tmaxAttempts: %s \n\t\tschedule: %s' % (bestFitness, bestParams['max_attempts'], type(bestParams['schedule']).__name__)) return bestCurve
def get_model(algorithm, max_iters): activation = "relu" print(algorithm) print(max_iters) if algorithm == "rh": return mlrose.NeuralNetwork(hidden_nodes = [10], activation = activation, algorithm = 'random_hill_climb', \ bias = True, is_classifier = True, early_stopping = True, restarts = 5, max_attempts =10, max_iters = max_iters, clip_max = 10, random_state = randomSeed) if algorithm == "ga": return mlrose.NeuralNetwork(hidden_nodes = [10], activation = activation, algorithm = 'genetic_alg', \ bias = True, is_classifier = True, early_stopping = True, max_attempts =10, max_iters = max_iters, clip_max = 10, mutation_prob = .10, random_state = randomSeed) if algorithm == "sa": return mlrose.NeuralNetwork(hidden_nodes = [10], activation = activation, algorithm = 'simulated_annealing', \ bias = True, is_classifier = True, early_stopping = True, max_attempts =10, max_iters = max_iters, clip_max = 10, schedule = mlrose.GeomDecay(), random_state = randomSeed) if algorithm == "gd": return mlrose.NeuralNetwork(hidden_nodes = [10], activation = activation, algorithm = 'gradient_descent', \ bias = True, is_classifier = True, early_stopping = True, max_attempts =10, max_iters = max_iters, clip_max = 10, random_state = randomSeed)
def nn_impl(): #iris_data = fetch_openml('iris') #X_whole, y_whole = iris_data['data'], iris_data['target'] sklearn_data = datasets.load_breast_cancer() x, y = sklearn_data.data, sklearn_data.target #x = preprocessing.scale(x) # Split the initial data xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.4, random_state=42) ### Analysis for RHC ### train_accuracy_scores = [] test_accuracy_scores = [] time_per_iteration_rhc = [] for i in range(1, 3000, 50): print(i) rhc_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='identity', algorithm='random_hill_climb', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, max_iters=i) start = time.time() rhc_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = rhc_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = rhc_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) time_per_iteration_rhc.append(time.time() - start) plt.figure() plt.plot(np.arange(1, 3000, 50), np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(np.arange(1, 3000, 50), np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('Iterations') plt.ylabel('Accuracy') plt.title('Accuracy vs. Iterations (RHC)') plt.legend() plt.savefig('testacc_iter_rhc.png') print("Finished RHC") ### Analysis for Simulated Annealing ### train_accuracy_scores = [] test_accuracy_scores = [] time_per_iteration_sa = [] for i in range(1, 3000, 50): print(i) sa_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='identity', algorithm='simulated_annealing', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, max_iters=i) start = time.time() sa_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = sa_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = sa_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) time_per_iteration_sa.append(time.time() - start) plt.figure() plt.plot(np.arange(1, 3000, 50), np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(np.arange(1, 3000, 50), np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('Iterations') plt.ylabel('Accuracy') plt.title('Accuracy vs. Iterations (SA)') plt.legend() plt.savefig('testacc_iter_SA.png') print("Finished SA") ### Analysis for Genetic Algorithms ### train_accuracy_scores = [] test_accuracy_scores = [] time_per_iteration_ga = [] for i in range(1, 3000, 50): print(i) ga_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='identity', algorithm='genetic_alg', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, max_iters=i) start = time.time() ga_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = ga_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = ga_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) time_per_iteration_ga.append(time.time() - start) plt.figure() plt.plot(np.arange(1, 3000, 50), np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(np.arange(1, 3000, 50), np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('Iterations') plt.ylabel('Accuracy') plt.title('Accuracy vs. Iterations (GA)') plt.legend() plt.savefig('testacc_iter_GA.png') print("Finished GA") ### Backpropogation (for comparison) ### train_accuracy_scores = [] test_accuracy_scores = [] time_per_iteration_bp = [] print("backprop start") for i in range(1, 3000, 50): print(i) bp_nn = MLPClassifier(hidden_layer_sizes=(50, ), activation='logistic', max_iter=i) # bp_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='identity', # algorithm='gradient_descent', # bias=False, is_classifier=True, # learning_rate = 0.6, clip_max=1, # max_attempts=1000, max_iters = i) start = time.time() bp_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = bp_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = bp_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) time_per_iteration_bp.append(time.time() - start) plt.figure() plt.plot(np.arange(1, 3000, 50), np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(np.arange(1, 3000, 50), np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('Iterations') plt.ylabel('Accuracy') plt.title('Accuracy vs. Iterations (Backpropogation)') plt.legend() plt.savefig('testacc_iter_bp.png') print("Finished Backprop") ### Plot runtimes for above ### plt.figure() plt.plot(np.arange(1, 3000, 50), np.array(time_per_iteration_rhc), label='RHC') plt.plot(np.arange(1, 3000, 50), np.array(time_per_iteration_sa), label='SA') plt.plot(np.arange(1, 3000, 50), np.array(time_per_iteration_ga), label='GA') plt.plot(np.arange(1, 3000, 50), np.array(time_per_iteration_ga), label='BP') plt.xlabel('Iterations') plt.ylabel('Training Time') plt.title('Training Time vs Iterations') plt.legend() plt.savefig('time_vs_iter.png') #### Hyperparameter Tuning - RHC #### ## Adjusting the number of random restarts ## train_accuracy_scores = [] test_accuracy_scores = [] for i in range(0, 500, 25): print(i) rhc_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='identity', algorithm='random_hill_climb', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, restarts=i) rhc_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = rhc_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = rhc_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) plt.figure() plt.plot(np.arange(0, 500, 25), np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(np.arange(0, 500, 25), np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('Restarts') plt.ylabel('Accuracy') plt.title('Accuracy vs. Number of Restarts (RHC)') plt.legend() plt.savefig('rhc_restarts.png') print("Finished RHC HP Tuning") #### Hyperparameter Tuning - SA #### ## Adjusting the type of scheduling ## train_accuracy_scores = [] test_accuracy_scores = [] # Referending sectiion 2.2 'Decay Schedules' here: # https://readthedocs.org/projects/mlrose/downloads/pdf/stable/ schedule_types = [ mlrose_hiive.ExpDecay(), mlrose_hiive.ArithDecay(), mlrose_hiive.GeomDecay() ] for st in schedule_types: print(st) sa_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='identity', algorithm='simulated_annealing', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, schedule=st) sa_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = sa_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = sa_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) plt.figure() plt.plot(['ExpDecay', 'ArithDecay', 'GeomDecay'], np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(['ExpDecay', 'ArithDecay', 'GeomDecay'], np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('Schedule Type') plt.ylabel('Accuracy') plt.title('Accuracy vs. Schedule Type (SA)') plt.legend() plt.savefig('sa_schedule_type.png') print("Finished SA HP Tuning") #### Hyperparameter Tuning - GA #### ## Adjusting the amount of mutation ## Used api as referenced in https://readthedocs.org/projects/mlrose/downloads/pdf/stable/ train_accuracy_scores = [] test_accuracy_scores = [] mutation_prob_array = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] for i in mutation_prob_array: ga_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='relu', algorithm='genetic_alg', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, mutation_prob=i) ga_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = ga_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = ga_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) plt.figure() plt.plot(mutation_prob_array, np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(mutation_prob_array, np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('mutation_prob') plt.ylabel('Accuracy') plt.title('Accuracy vs. Iterations (GA - mutation_prob experimentation)') plt.legend() plt.savefig('ga_mutation.png') print("Finished GA mutation experimentation") ## Adjusting the population size ## Used api as referenced in https://readthedocs.org/projects/mlrose/downloads/pdf/stable/ train_accuracy_scores = [] test_accuracy_scores = [] pop_size_array = [100, 200, 300, 400, 500] for i in pop_size_array: ga_nn = mlrose_hiive.NeuralNetwork(hidden_nodes=[2], activation='relu', algorithm='genetic_alg', bias=False, is_classifier=True, learning_rate=0.6, clip_max=1, max_attempts=1000, pop_size=i) ga_nn.fit(xtrain, ytrain) # Train set analysis predictions_train = ga_nn.predict(xtrain) accuracy_score_train = accuracy_score(ytrain, predictions_train) train_accuracy_scores.append(accuracy_score_train) # Test set analysis predictions_test = ga_nn.predict(xtest) accuracy_score_test = accuracy_score(ytest, predictions_test) test_accuracy_scores.append(accuracy_score_test) plt.figure() plt.plot(pop_size_array, np.array(train_accuracy_scores), label='Train Accuracy') plt.plot(pop_size_array, np.array(test_accuracy_scores), label='Test Accuracy') plt.xlabel('pop_size') plt.ylabel('Accuracy') plt.title('Accuracy vs. Iterations (GA - pop_size experimentation)') plt.legend() plt.savefig('ga_popsize.png') print("Finished GA pop_size experimentation")
def NN_SA(file_name, classifier_col): X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col) activation = 'relu' learning_rate = [0.01, 0.1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] algorithim = 'simulated_annealing' iters = [ 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000 ] nodes = [128, 128, 128, 128] temperatures = [0.001, 0.01] decay_rates = [0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99] outcomes = [] max_attempts = [10, 50, 100, 200, 500, 1000] clips = [5, 10, 100, 1000, 10000, 100000] csv_file = 'NN_SA-itertests.csv' act = 'relu' lr = 10 itera = 10000 temp = 10000 dec = 0.92 ma = 50 clip = 10 while 1 == 1: iters_outs = {} for iter_test in iters: start = time.time() print(algorithim, act, lr, iter_test, 'GeomDecay', temp, dec, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=iter_test, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) train_time = time.time() - start print('Train time', train_time) start = time.time() y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_train_query_time = time.time() - start print('y_train_query_time', y_train_query_time) start = time.time() y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) y_test_query_time = time.time() - start print('y_test_query_time', y_test_query_time) nn_loss = nn_model.loss print('loss', nn_loss) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = iter_test outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome[ 'runtime'] = train_time + y_train_query_time + y_test_query_time outcome['Train time'] = train_time outcome['y_train_query_time'] = y_train_query_time outcome['y_test_query_time'] = y_test_query_time outcome['loss'] = nn_loss outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) iters_outs[iter_test] = y_test_roc old_val = itera itera = max(iters_outs, key=iters_outs.get) print('best iter', itera, 'old', old_val) raise SystemExit(0) temp_outs = {} for temp_test in temperatures: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp_test, dec, ma, clip) nn_model = mlrose.NeuralNetwork( hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay(init_temp=temp_test, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp_test outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) temp_outs[temp_test] = y_test_roc old_temp = temp temp = max(temp_outs, key=temp_outs.get) print('best temp', temp, 'old', old_temp) decay_outs = {} for decay_test in decay_rates: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp, decay_test, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=decay_test), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = decay_test outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) decay_outs[decay_test] = y_test_roc old_val = dec dec = max(decay_outs, key=decay_outs.get) print('best decay', dec, 'old', old_val) clips_outs = {} for clip_test in clips: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp, dec, ma, clip_test) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip_test, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip_test outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) clips_outs[clip_test] = y_test_roc old_val = clip clip = max(clips_outs, key=clips_outs.get) print('best clip', clip, 'old', old_val) maxa_outs = {} for maxa_test in max_attempts: start = time.time() print(algorithim, act, lr, itera, 'GeomDecay', temp, dec, maxa_test, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr, early_stopping=True, clip_max=clip, max_attempts=maxa_test, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = maxa_test outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) maxa_outs[maxa_test] = y_test_roc old_val = ma ma = max(maxa_outs, key=maxa_outs.get) print('best ma', ma, 'old', old_val) lr_outs = {} for lr_test in learning_rate: start = time.time() print(algorithim, act, lr_test, itera, 'GeomDecay', temp, dec, ma, clip) nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, max_iters=itera, algorithm=algorithim, schedule=mlrose.GeomDecay( init_temp=temp, decay=dec), bias=True, is_classifier=True, learning_rate=lr_test, early_stopping=True, clip_max=clip, max_attempts=ma, random_state=1, curve=True) nn_model.fit(X_train, y_train) y_train_pred = nn_model.predict(X_train) y_train_roc = roc_auc_score(y_train, y_train_pred, multi_class="ovr", average="weighted") print('y_train_roc', y_train_roc) y_test_pred = nn_model.predict(X_test) y_test_roc = roc_auc_score(y_test, y_test_pred, multi_class="ovr", average="weighted") print('y_test_roc', y_test_roc) runtime = time.time() - start print('curr run time', time.time() - start) outcome = {} outcome['schedule'] = 'GeomDecay' outcome['activation'] = act outcome['learning_rate'] = lr_test outcome['max_iters'] = itera outcome['temperatures'] = temp outcome['decay_rates'] = dec outcome['max_attempts'] = ma outcome['clip'] = clip outcome['y_train_roc'] = y_train_roc outcome['y_test_roc'] = y_test_roc outcome['runtime'] = runtime outcomes.append(outcome) pd.DataFrame(outcomes).to_csv(csv_file) lr_outs[lr_test] = y_test_roc old_lr = lr lr = max(lr_outs, key=lr_outs.get) print('best lr', lr, 'old', old_lr)
prob_length = 50 np.random.seed(0) coords_list = [] for n in range(prob_length): coords_list.append(np.random.rand(2)) fitness = mlrose.TravellingSales(coords=coords_list) problem = mlrose.TSPOpt(prob_length, fitness) RANDOM_SEED = 42 MAX_ATTEMPTS = 200 #%% tuning for SA curve_list = [] decays = [0.999, 0.99, 0.9] for d in decays: schedule = mlrose.GeomDecay(decay=d) _, _, curve = mlrose.simulated_annealing( problem, schedule=schedule, max_attempts=MAX_ATTEMPTS, max_iters=3000, curve=True, random_state=RANDOM_SEED, ) curve_list.append(curve) df = 1 / pd.DataFrame(curve_list).transpose() df.columns = decays df.plot() plt.xlabel("Iteration") plt.ylabel("Fitness")
y_test_hot = oh.transform(y_test.reshape(-1, 1)).todense().astype(int) # lrs = [0.002] # 6 # lrs = [0.002, 0.004, 0.02, 0.06, 0.2, 0.8] # 6 lrs = [0.003, 0.05, 0.7] # 6 # mil = [100] # 4 # mil = [100, 500, 1000, 2500] # 4 mil = [145, 435, 1300] # 4 # acti = ['tanh'] acti = ['relu', 'sigmoid', 'tanh'] NNparams = np.array(np.meshgrid(lrs,mil,acti),dtype=object).T.reshape(-1,3) # rsts = [10] rsts = [7, 12] # rsts = [5, 10, 15] # schs = [mlr.ArithDecay(15)] schs = [mlr.ArithDecay(25), mlr.ExpDecay(25), mlr.GeomDecay(25)] # schs = [mlr.ArithDecay(15), mlr.ExpDecay(15), mlr.GeomDecay(15), # mlr.ArithDecay(175), mlr.ExpDecay(175), mlr.GeomDecay(175)] # pops = [225] pops = [150, 225] # pops = [75, 150, 225] # muts = [0.25] muts = [0.33, 0.66] # muts = [0.25, 0.5, 0.75] gaParams = np.array(np.meshgrid(pops, muts)).T.reshape(-1,2) experiments = [] for lr, mi, act in NNparams: for rst in rsts: experiments.append(mlr.NeuralNetwork(hidden_nodes=[7,5], activation=act,
def NN_GA(file_name, classifier_col): # HPs shared by all nodes = [128, 128, 128, 128] act = 'relu' seed = 1 # GD HPs gd_algo = 'gradient_descent' gd_lr = 0.00000009 gd_iter = 10000 gd_ma = 50 gd_clip = 5 # RHC HPs rhc_algo = 'random_hill_climb' rhc_lr = 8 rhc_iter = 10000 rhc_restarts = 10 rhc_ma = 100 rhc_clip = 100 # SA HPs sa_algo = 'simulated_annealing' sa_lr = 10 sa_iter = 10000 sa_temp = 10000 sa_decay = 0.92 sa_ma = 50 sa_clip = 10 # GA HPs ga_algo = 'genetic_alg' ga_lr = 5 ga_iter = 100 ga_pop = 1500 ga_mut = 0.1 ga_ma = 100 ga_clip = 5 X_train, X_test, y_train, y_test = util.data_load(file_name, classifier_col, random_seed=seed) # Best GD Algorithm print('Training GD NN') gd_start = time.time() gd_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=gd_algo, max_iters=gd_iter, learning_rate=gd_lr, clip_max=gd_clip, max_attempts=gd_ma) gd_nn_model.fit(X_train, y_train) print('gd loss:', gd_nn_model.loss) gd_train_time = time.time() - gd_start print('gd_train_time:', gd_train_time) start = time.time() gd_y_train_pred = gd_nn_model.predict(X_train) gd_y_train_roc = roc_auc_score(y_train, gd_y_train_pred, multi_class="ovr", average="weighted") gd_y_train_query_time = time.time() - start print('gd_y_train_roc', gd_y_train_roc, 'gd_y_train_roc: ', gd_y_train_query_time) start = time.time() gd_y_test_pred = gd_nn_model.predict(X_test) gd_y_test_roc = roc_auc_score(y_test, gd_y_test_pred, multi_class="ovr", average="weighted") gd_y_test_query_time = time.time() - start print('gd_y_test_roc', gd_y_test_roc, 'gd_y_test_query_time: ', gd_y_test_query_time) # Best RHC Algorithm print('Training RHC NN') rhc_start = time.time() rhc_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=rhc_algo, max_iters=rhc_iter, learning_rate=rhc_lr, clip_max=rhc_clip, max_attempts=rhc_ma, restarts=rhc_restarts) rhc_nn_model.fit(X_train, y_train) print('rhc loss:', rhc_nn_model.loss) rhc_train_time = time.time() - rhc_start print('rhc_train_time:', rhc_train_time) start = time.time() rhc_y_train_pred = rhc_nn_model.predict(X_train) rhc_y_train_roc = roc_auc_score(y_train, rhc_y_train_pred, multi_class="ovr", average="weighted") rhc_y_train_query_time = time.time() - start print('rhc_y_train_roc', rhc_y_train_roc, 'rhc_y_train_roc: ', rhc_y_train_query_time) start = time.time() rhc_y_test_pred = rhc_nn_model.predict(X_test) rhc_y_test_roc = roc_auc_score(y_test, rhc_y_test_pred, multi_class="ovr", average="weighted") rhc_y_test_query_time = time.time() - start print('rhc_y_test_roc', rhc_y_test_roc, 'rhc_y_test_query_time: ', rhc_y_test_query_time) # Best SA Algorithm print('Training SA NN') sa_start = time.time() sa_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=sa_algo, max_iters=sa_iter, learning_rate=sa_lr, clip_max=sa_clip, max_attempts=sa_ma, schedule=mlrose.GeomDecay( init_temp=sa_temp, decay=sa_decay)) sa_nn_model.fit(X_train, y_train) print('sa loss:', sa_nn_model.loss) sa_train_time = time.time() - sa_start print('sa_train_time:', sa_train_time) start = time.time() sa_y_train_pred = sa_nn_model.predict(X_train) sa_y_train_roc = roc_auc_score(y_train, sa_y_train_pred, multi_class="ovr", average="weighted") sa_y_train_query_time = time.time() - start print('sa_y_train_roc', sa_y_train_roc, 'sa_y_train_roc: ', sa_y_train_query_time) start = time.time() sa_y_test_pred = sa_nn_model.predict(X_test) sa_y_test_roc = roc_auc_score(y_test, sa_y_test_pred, multi_class="ovr", average="weighted") sa_y_test_query_time = time.time() - start print('sa_y_test_roc', sa_y_test_roc, 'sa_y_test_query_time: ', sa_y_test_query_time) # Best Genetic Algorithm print('Training GA NN') ga_start = time.time() ga_nn_model = mlrose.NeuralNetwork(hidden_nodes=nodes, activation=act, random_state=seed, bias=True, is_classifier=True, early_stopping=True, curve=True, algorithm=ga_algo, max_iters=ga_iter, learning_rate=ga_lr, clip_max=ga_clip, max_attempts=ga_ma, pop_size=ga_pop, mutation_prob=ga_mut) ga_nn_model.fit(X_train, y_train) print('ga loss:', ga_nn_model.loss) ga_train_time = time.time() - ga_start print('ga_train_time:', ga_train_time) start = time.time() ga_y_train_pred = ga_nn_model.predict(X_train) ga_y_train_roc = roc_auc_score(y_train, ga_y_train_pred, multi_class="ovr", average="weighted") ga_y_train_query_time = time.time() - start print('ga_y_train_roc', ga_y_train_roc, 'ga_y_train_roc: ', ga_y_train_query_time) start = time.time() ga_y_test_pred = ga_nn_model.predict(X_test) ga_y_test_roc = roc_auc_score(y_test, ga_y_test_pred, multi_class="ovr", average="weighted") ga_y_test_query_time = time.time() - start print('ga_y_test_roc', ga_y_test_roc, 'ga_y_test_query_time: ', ga_y_test_query_time) # Plot Loss Curves plt.figure() plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve') plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve') plt.plot(gd_nn_model.fitness_curve, label='GD Loss Curve') plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve') plt.title("Neural Network Loss Curves") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.xscale('log') plt.savefig("Images\\Neural Network Loss Curves") plt.show() # Plot Loss Curves (GD Inverted) plt.figure() gd_curve = gd_nn_model.fitness_curve inverted_gd_curve = np.array(gd_curve) * -1 plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve') plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve') plt.plot(inverted_gd_curve, label='GD Loss Curve') plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve') plt.title("Neural Network Loss Curves") plt.xlabel('Iterations') plt.ylabel('Loss') plt.grid(True) plt.legend() plt.xscale('log') plt.savefig("Images\\Neural Network Loss Curves-inverted GD") plt.show() # Plot Loss Curves - No GD plt.figure() plt.plot(ga_nn_model.fitness_curve, label='GA Loss Curve') plt.plot(sa_nn_model.fitness_curve, label='SA Loss Curve') plt.plot(rhc_nn_model.fitness_curve, label='RHC Loss Curve') plt.title("Neural Network Loss Curves") plt.xlabel('Iterations') plt.ylabel('Loss') plt.ylim(bottom=0) plt.grid(True) plt.legend() plt.xscale('log') plt.savefig("Images\\Neural Network Loss Curves - No GD") plt.show()
attempts = max_it * 3 / 4 rhc_fitnesses = [] sa_fitnesses = [] ga_fitnesses = [] m_fitnesses = [] rhc_times = [] sa_times = [] ga_times = [] m_times = [] test_range = [3, 4, 5, 6] final_prob_len = 7 fitness_cust = mlr.CustomFitness(mults_of_2) schedule = mlr.GeomDecay(init_temp=10, decay=0.95, min_temp=0.01) print(f"Attempts: {attempts}") print(f"Max Iterations: {max_it}") print(f"Problem Sizes: {test_range}") print(f"Last Problem Size: {final_prob_len}\n\n") part2_time = 0.0 print(f"\n######### PART 2 #########\n") for i in test_range: start = time.time() init = np.random.choice(2**(i + 1), size=i, replace=False) print(f"Running for subproblem size: {i}\n Initialization: {init}")
y2, test_size=0.3, random_state=42) #transform data using StandardScaler scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) hidden_layers = [12, 6, 4, 2] models = {} algos = { 'SA': { 'algorithm': ['simulated_annealing'], 'schedule': [ mlrose.GeomDecay(init_temp=1, decay=0.99), mlrose.GeomDecay(init_temp=100, decay=0.99), mlrose.GeomDecay(init_temp=1000, decay=0.99), mlrose.ExpDecay(init_temp=1, exp_const=0.05), mlrose.ExpDecay(init_temp=100, exp_const=0.05), mlrose.ExpDecay(init_temp=1000, exp_const=0.05), ], 'max_iters': [3000], 'learning_rate': [0.5], 'activation': ['relu', 'sigmoid', 'tanh'], 'clip_max': [10.0] }, 'GA': { 'algorithm': ['genetic_alg'], 'pop_size': [100, 500], 'mutation_prob': [0.3, 0.5],
def run(self): self.loadDataset() self.split_data() # Hyperparams - Trial One # h_params = { # 'learning_rates': np.linspace(0.1, .5, 10), # 'max_iters': 100, # 'activation_functions': ['relu'], # 'hidden_layers': [[5], [10], [5,5]], # 'restarts': 8 # } # # Hyperparams - Trial Two # h_params = { # 'learning_rates': np.linspace(0.3, .5, 10), # 'max_iters': 100, # 'activation_functions': ['relu'], # 'hidden_layers': [[5]], # # RHC # 'restarts': 8 # } # # rhc_curve = self.runTrial('random_hill_climb', **h_params) # # Hyperparams - Trial Three # h_params = { # 'learning_rates': np.linspace(0.3, .5, 10), # 'max_iters': 100, # 'activation_functions': ['relu'], # # 'hidden_layers': [[5], [10], [5,5]], # 'hidden_layers': [[5]], # 'restarts': 0, # # GA # 'pop_sizes': [10, 20, 50, 100], # 'mutation_probs': np.linspace(0.1, 1, 5) # } # ga_curve = self.runTrial('genetic_alg', **h_params) # # Hyperparams - Trial Three # h_params = { # 'learning_rates': np.linspace(0.2, .5, 10), # 'max_iters': 1000, # 'activation_functions': ['relu'], # 'hidden_layers': [[5]], # 'restarts': 8, # # GA # 'schedules': [mlrose.GeomDecay(), mlrose.ExpDecay()] # } # sa_curve = self.runTrial('simulated_annealing', **h_params) # Optimal Parameters Below rhc_params = { 'learning_rates': [0.43333333333333335], 'max_iters': 400, 'activation_functions': ['relu'], 'hidden_layers': [[5]], 'restarts': 8 } ga_params = { 'learning_rates': [0.3], 'max_iters': 300, 'activation_functions': ['relu'], 'hidden_layers': [[5]], 'restarts': 8, # GA 'pop_sizes': [200], 'mutation_probs': [0.1] } sa_params = { 'learning_rates': [0.43333333333333335], 'max_iters': 1000, 'activation_functions': ['relu'], 'hidden_layers': [[5]], 'restarts': 8, 'schedules': [mlrose.GeomDecay()] } rhc_curve = self.runTrial('random_hill_climb', **rhc_params) sa_curve = self.runTrial('simulated_annealing', **sa_params) ga_curve = self.runTrial('genetic_alg', **ga_params) a = np.array(sa_curve) b = np.array(ga_curve) c = np.array(rhc_curve) maxLen = max((len(a), len(b), len(c))) arr = np.zeros((3, maxLen)) arr[0, :len(a)] = a arr[1, :len(b)] = b arr[1, :len(c)] = c arr[0, len(a):maxLen] = a[-1] arr[1, len(b):maxLen] = b[-1] arr[1, len(c):maxLen] = c[-1] saveDir = os.path.join(self.savePath, '%s.png' % 'NN Weight Training') graph.plotPart2(arr, saveDir, title='NN Weight Training', isMaximizing=False, xmax=maxLen+5)
def runTrial(self, algorithm, **h_params): # Initialize neural network object and fit object print(h_params) activation_functions = h_params['activation_functions'] hidden_layers = h_params['hidden_layers'] learning_rates = h_params['learning_rates'] restarts = h_params['restarts'] max_iters = h_params['max_iters'] # pop_sizes = h_params['pop_sizes'] # mutation_probs = h_params['mutation_probs'] if algorithm == 'simulated_annealing': schedules = h_params['schedules'] else: schedules = [mlrose.GeomDecay()] if algorithm == 'genetic_alg': pop_sizes = h_params['pop_sizes'] mutation_probs = h_params['mutation_probs'] else: mutation_probs = [0.1] pop_sizes = [200] csvFile = open(os.path.join(self.savePath, algorithm+'_output.csv'), 'w') header = 'Algorithm, Activation Function, Learning Rate, Restarts, Hidden Layers, Training Accuracy, Validation Accuracy, Training Time\n' csvFile.write(header) best_validation_accuracy = 0 best_training_accuracy = 0 best_params = None for schedule in schedules: for layers in hidden_layers: for learning_rate in learning_rates: for pop_size in pop_sizes: for mutation in mutation_probs: paramString = '%s, activation, %s, learning_rate, %f, restarts, %i, hidden_layers, %s, pop_size, %s, mutation_prob, %s ' %(algorithm, 'relu', learning_rate, restarts, layers, pop_size, mutation) log.info(paramString) # print('Learning rate: ', learning_rate) nn_model1 = mlrose.NeuralNetwork(hidden_nodes = layers, activation = 'relu', algorithm = algorithm, restarts=restarts, max_iters = max_iters, bias = True, is_classifier = True, learning_rate = learning_rate, early_stopping = True, clip_max = 5, curve=True, schedule=schedule, pop_size=pop_size, mutation_prob=mutation, max_attempts = 100, random_state = 3) start = time.process_time() nn_model1.fit(self.X_train_scaled, self.y_train_hot) elapsed = time.process_time() - start # log.info('\tElapsed time, %s' %elapsed) # Predict labels for train set and assess accuracy y_train_pred = nn_model1.predict(self.X_train_scaled) y_train_accuracy = accuracy_score(self.y_train_hot, y_train_pred) # Predict labels for test set and assess accuracy y_validate_pred = nn_model1.predict(self.X_validate_scaled) y_validate_accuracy = accuracy_score(self.y_validate_hot, y_validate_pred) # Predict labels for train set and assess accuracy y_test_pred = nn_model1.predict(self.X_test_scaled) y_test_accuracy = accuracy_score(self.y_test_hot, y_test_pred) log.info('\tTraining Accuracy,\t %f' %(y_train_accuracy)) log.info('\tValidation Accuracy,\t %f'%y_validate_accuracy) log.info('\Test Accuracy,\t %f'%y_test_accuracy) log.info('\tTraining Time,\t\t %f' %elapsed) esc_layers = ('%s' %layers).replace(",", ";") vals = '%s,%s,%s,%s,%s,%s,%s,%s,\n' %(algorithm, 'relu', learning_rate, restarts, esc_layers, y_train_accuracy, y_validate_accuracy, elapsed) csvFile.write(vals) # confusion = confusion_matrix(self.y_train_hot, y_train_pred) if (y_validate_accuracy > best_validation_accuracy): best_validation_accuracy = y_validate_accuracy best_training_accuracy = y_train_accuracy best_params = nn_model1.get_params() best_curve = nn_model1.fitness_curve best_weights = nn_model1.fitted_weights log.info('\t\t%s - Best validation score: %f, training score: %f, Best Params: %s, Best Weights: %s' %(algorithm, best_validation_accuracy, best_training_accuracy, best_params, best_weights)) csvFile.write('\nAlgorithm, Best validation score, Training Score, Best Params,\n') esc_params = ('%s' %best_params).replace(",", ";") csvFile.write('\n%s, %f, %f, %s' %(algorithm, best_validation_accuracy, best_training_accuracy, esc_params)) csvFile.close() return best_curve
def main(): ## SET SOME PARAMS TO USE GLOBALLY max_iters_list = [50, 100, 1000] #,32,64,128,256,512,1024] max_iters_list_full = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] rand_list = [1, 11, 22] #,44,55,66,77,88,99] rand_list_full = [0, 11, 22, 33, 44, 55, 66, 77, 88, 99] input_location = 'data/' output_location = 'outputs/' chart_output_location = 'charts/' prefix = '5th_' ## DEFINE PROBLEMS TO SOLVE # Traveling Salesman Problem (TSP) space_length = 1000 cities_cnt = 200 coords_list, x, y = create_TSP(space_length, cities_cnt, return_lists_too=True) plt.plot(x, y, 'o') plt.savefig(chart_output_location + 'TPS_visual' + '.png') fitness_coords = mlrose.TravellingSales(coords=coords_list) problem_TSP = mlrose.TSPOpt(length=len(coords_list), fitness_fn=fitness_coords, maximize=False) # 4 Peaks t_pct = 0.1 length = 200 fitness_4_peaks = mlrose.FourPeaks(t_pct=t_pct) problem_4P = mlrose.DiscreteOpt(length=length, fitness_fn=fitness_4_peaks, maximize=True, max_val=2) problem_4P_small = mlrose.DiscreteOpt(length=50, fitness_fn=fitness_4_peaks, maximize=True, max_val=2) problem_4P_big = mlrose.DiscreteOpt(length=1000, fitness_fn=fitness_4_peaks, maximize=True, max_val=2) # Continuous Peaks t_pct = 0.1 length = 200 fitness_cont_peaks = mlrose.ContinuousPeaks(t_pct=t_pct) problem_cont_peaks = mlrose.DiscreteOpt(length=length, fitness_fn=fitness_cont_peaks, maximize=True, max_val=2) # Flip Flop length = 200 fitness_FF = mlrose.FlipFlop() problem_FF = mlrose.DiscreteOpt(length=length, fitness_fn=fitness_FF, maximize=True, max_val=2) problem_FF_small = mlrose.DiscreteOpt(length=50, fitness_fn=fitness_FF, maximize=True, max_val=2) problem_FF_big = mlrose.DiscreteOpt(length=1000, fitness_fn=fitness_FF, maximize=True, max_val=2) # Knapsack length = 200 weights, values = create_Knapsack(length) weights_big, values_big = create_Knapsack(1000) weights_small, values_small = create_Knapsack(50) fitness_KS = mlrose.Knapsack(weights, values, max_weight_pct=0.65) fitness_KS_big = mlrose.Knapsack(weights_big, values_big, max_weight_pct=0.65) fitness_KS_small = mlrose.Knapsack(weights_small, values_small, max_weight_pct=0.65) problem_KS = mlrose.DiscreteOpt(length=length, fitness_fn=fitness_KS, maximize=True, max_val=2) problem_KS_big = mlrose.DiscreteOpt(length=1000, fitness_fn=fitness_KS_big, maximize=True, max_val=2) problem_KS_small = mlrose.DiscreteOpt(length=50, fitness_fn=fitness_KS_small, maximize=True, max_val=2) dict_of_param_dict = {} dict_of_param_dict['GA'] = { 'pop_size': [100, 200], #,1000], 'mutation_prob': [0.5, 0.1, 0.2], 'max_attempts': [5, 10, 30], 'max_iters': max_iters_list, 'random_state': rand_list } dict_of_param_dict['RHC'] = { 'max_attempts': [30, 50, 100], #[5,10,20,50] 'restarts': [5, 10, 20], #[0,1,2,5] 'max_iters': max_iters_list, 'random_state': rand_list } dict_of_param_dict['SA'] = { 'max_attempts': [10, 50, 100], 'init_temp': [1.0, 10.0, 0.5, 20, 100, 1000], 'decay': [0.99, 0.8, 0.5], 'max_iters': max_iters_list, 'random_state': rand_list } dict_of_param_dict['MIMIC'] = { 'pop_size': [100, 150], 'keep_pct': [0.5, 0.2], 'max_attempts': [10], 'max_iters': [100], 'random_state': rand_list } MIMIC_FF = { 'pop_size': 100, 'keep_pct': 0.5, 'max_attempts': 30, 'max_iters': [2, 4, 8, 16, 32, 64, 128], ## put full list here before uploading 'random_state': [0, 11, 22, 33, 44] } MIMIC_4P = { 'pop_size': 150, 'keep_pct': 0.2, 'max_attempts': 30, 'max_iters': [2, 4, 8, 16, 32, 64, 128], ## put full list here before uploading 'random_state': [0, 11, 22, 33, 44] } MIMIC_KS = { 'pop_size': 150, 'keep_pct': 0.5, 'max_attempts': 30, 'max_iters': [2, 4, 8, 16, 32, 64, 128], ## put full list here before uploading 'random_state': [0, 11, 22, 33, 44] } MIMIC_CP = { 'pop_size': 200, 'keep_pct': 0.2, 'max_attempts': 30, 'max_iters': [2, 4, 8, 16, 32, 64, 128], ## put full list here before uploading 'random_state': [0, 11, 22, 33, 44] } GA_FF = { 'pop_size': 200, #,1000], 'mutation_prob': 0.5, 'max_attempts': 30, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } MIMIC_FF2 = { 'pop_size': [100], 'keep_pct': [0.5], 'max_attempts': [30, 50], 'max_iters': [64], 'random_state': [55] #,66,77,88,99] } print("starting MIMIC FF") # GETTING MIMIC FF RESULTS print("starting MIMIC FF...") ''' ## Started running at 3am results_df, curve_output_list = fitness_by_iter('MIMIC', problem_FF, MIMIC_FF['max_iters'], MIMIC_FF['random_state']\ , pop_size=MIMIC_FF['pop_size'], max_attempts=MIMIC_FF['max_attempts'], curve=True, keep_pct=MIMIC_FF['keep_pct']) results_df.to_csv(output_location + 'final_MIMIC_FF_attempt_3am.csv') results_df, curve_output_list = fitness_by_iter('MIMIC', problem_4P, MIMIC_4P['max_iters'], MIMIC_4P['random_state']\ , pop_size=MIMIC_4P['pop_size'], max_attempts=MIMIC_4P['max_attempts'], curve=True, keep_pct=MIMIC_4P['keep_pct']) results_df.to_csv(output_location + 'final_MIMIC_4P_attempt_3am.csv') results_df, curve_output_list = fitness_by_iter('MIMIC', problem_KS, MIMIC_KS['max_iters'], MIMIC_KS['random_state']\ , pop_size=MIMIC_KS['pop_size'], max_attempts=MIMIC_KS['max_attempts'], curve=True, keep_pct=MIMIC_KS['keep_pct']) results_df.to_csv(output_location + 'final_MIMIC_KS_attempt_3am.csv') results_df, curve_output_list = fitness_by_iter('MIMIC', problem_cont_peaks, MIMIC_CP['max_iters'], MIMIC_CP['random_state']\ , pop_size=MIMIC_CP['pop_size'], max_attempts=MIMIC_CP['max_attempts'], curve=True, keep_pct=MIMIC_CP['keep_pct']) results_df.to_csv(output_location + 'final_MIMIC_CP_attempt_3am.csv') ''' ## USED FOR GRID SEARCHING PARAMETERS FOR RO ON 3 PROBLEMS GA_params_dict = get_params_for_grid_search('GA', max_iters_list=[200]) print("Here are my GA params for grid search: ", GA_params_dict) SA_params_dict = get_params_for_grid_search('SA', max_iters_list=max_iters_list) print("Here are my SA params for grid search: ", SA_params_dict) RHC_params_dict = get_params_for_grid_search('RHC', max_iters_list=max_iters_list) print("Here are my RHC params for grid search: ", RHC_params_dict) MIMIC_params_dict = get_params_for_grid_search( 'MIMIC', max_iters_list=max_iters_list) print("Here are my MIMIC params for grid search: ", MIMIC_params_dict) #grid_search_MIMIC = MIMIC_best_params(problem_TPS, MIMIC_params_dict, inverse_fitness=False) #grid_search_MIMIC.to_csv(output_location + 'grid_search_MIMIC.csv') ''' grid_search_GA = GA_best_params(problem_FF, GA_params_dict, inverse_fitness=False) grid_search_GA.to_csv(output_location + prefix + 'grid_search_GA_FF_really.csv') print("finished GA") grid_search_MIMIC = MIMIC_best_params(problem_FF, MIMIC_params_dict, inverse_fitness=False) grid_search_MIMIC.to_csv(output_location + prefix + 'grid_search_MIMIC_FF_really.csv') ''' print("finished MIMIC FF") print("Doing GA rn") #results_df, curve_output_list = fitness_by_iter('GA', problem_FF, GA_FF['max_iters'], GA_FF['random_state']\ #, pop_size=GA_FF['pop_size'], max_attempts=GA_FF['max_attempts'], mutation_prob=GA_FF['mutation_prob'],curve=True) #results_df.to_csv(output_location + 'final_MIMIC_FF_attempt_1am.csv') print("finished GA") ''' GRID SEARCHING print("Starting grid search for RHC") grid_search_RHC = RHC_best_params(problem_TSP, RHC_params_dict, inverse_fitness=False) grid_search_RHC.to_csv(output_location + prefix +'grid_search_RHC_TSP.csv') grid_search_RHC = RHC_best_params(problem_FF, RHC_params_dict, inverse_fitness=False) grid_search_RHC.to_csv(output_location + prefix + 'grid_search_RHC_FF.csv') grid_search_RHC = RHC_best_params(problem_cont_peaks, RHC_params_dict, inverse_fitness=False) grid_search_RHC.to_csv(output_location + prefix + 'grid_search_RHC_cont_peaks.csv') grid_search_RHC = RHC_best_params(problem_4P, RHC_params_dict, inverse_fitness=False) grid_search_RHC.to_csv(output_location + prefix + 'grid_search_RHC_4P.csv') print("Starting grid search for SA") grid_search_SA = SA_best_params(problem_TSP, SA_params_dict, inverse_fitness=False) grid_search_SA.to_csv(output_location + prefix + 'grid_search_SA_TSP.csv') grid_search_SA = SA_best_params(problem_FF, SA_params_dict, inverse_fitness=False) grid_search_SA.to_csv(output_location + prefix + 'grid_search_SA_FF.csv') grid_search_SA = SA_best_params(problem_cont_peaks, SA_params_dict, inverse_fitness=False) grid_search_SA.to_csv(output_location + prefix + 'grid_search_SA_cont_peaks.csv') grid_search_SA = SA_best_params(problem_4P, SA_params_dict, inverse_fitness=False) grid_search_SA.to_csv(output_location + prefix + 'grid_search_SA_4P.csv') print("Starting grid search for GA") grid_search_GA = GA_best_params(problem_TSP, GA_params_dict, inverse_fitness=False) grid_search_GA.to_csv(output_location + prefix + 'grid_search_GA_TSP.csv') grid_search_GA = GA_best_params(problem_FF, GA_params_dict, inverse_fitness=False) grid_search_GA.to_csv(output_location + prefix + 'grid_search_GA_FF.csv') grid_search_GA = GA_best_params(problem_cont_peaks, GA_params_dict, inverse_fitness=False) grid_search_GA.to_csv(output_location + prefix + 'grid_search_GA_cont_peaks.csv') grid_search_GA = GA_best_params(problem_4P, GA_params_dict, inverse_fitness=False) grid_search_GA.to_csv(output_location + prefix + 'grid_search_GA_4P.csv') ''' ''' print("Starting grid search for MIMIC") grid_search_MIMIC = MIMIC_best_params(problem_FF, MIMIC_params_dict, inverse_fitness=False) grid_search_MIMIC.to_csv(output_location + prefix + 'grid_search_MIMIC_FF.csv') #grid_search_MIMIC = MIMIC_best_params(problem_cont_peaks, MIMIC_params_dict, inverse_fitness=False) #grid_search_MIMIC.to_csv(output_location + prefix + 'grid_search_MIMIC_cont_peaks.csv') grid_search_MIMIC = MIMIC_best_params(problem_4P, MIMIC_params_dict, inverse_fitness=False) grid_search_MIMIC.to_csv(output_location + prefix + 'grid_search_MIMIC_4P.csv') #grid_search_MIMIC = MIMIC_best_params(problem_TSP, MIMIC_params_dict, inverse_fitness=False) #grid_search_MIMIC.to_csv(output_location + 'grid_search_MIMIC_TSP.csv') print("Finished MIMIC grid searches") print("Starting grid search for Knapsack") #grid_search_MIMIC = MIMIC_best_params(problem_KS, MIMIC_params_dict, inverse_fitness=False) #grid_search_MIMIC.to_csv(output_location + prefix + 'grid_search_MIMIC_KS.csv') #grid_search_GA = GA_best_params(problem_KS, GA_params_dict, inverse_fitness=False) #grid_search_GA.to_csv(output_location + prefix + 'grid_search_GA_KS.csv') grid_search_SA = SA_best_params(problem_KS, SA_params_dict, inverse_fitness=False) grid_search_SA.to_csv(output_location + prefix + 'grid_search_SA_KS.csv') grid_search_RHC = RHC_best_params(problem_KS, RHC_params_dict, inverse_fitness=False) grid_search_RHC.to_csv(output_location + prefix + 'grid_search_RHC_KS.csv') ''' ## Fitting MIMIC separately and with fewer iterations for all except the FF as run time is so long for MIMIC max = 128 ''' MIMIC CURVE FOR CHARTS ##### Started (again) at 8am ###### print("Fitting for MIMIC using the 'curve=True' functionality") print("First for KS") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_KS, pop_size=100, keep_pct=0.5, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_KS_full_curve.csv') df2.to_csv(output_location+'MIMIC_KS_short_curve.csv') print("Finished KS") print("Next for 4 Peaks") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_4P, pop_size=150, keep_pct=0.2, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_4P_full_curve.csv') df2.to_csv(output_location+'MIMIC_4P_short_curve.csv') print("Finished 4 Peaks") print("Next for 4 Peaks with 100 and 0.5") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_4P, pop_size=100, keep_pct=0.5, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_4P_pop100_keep50_full_curve.csv') df2.to_csv(output_location+'MIMIC_4P_pop100_keep50_short_curve.csv') print("Finished 4 Peaks") print("Next for 4 Peaks with 100 and 0.2") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_4P, pop_size=100, keep_pct=0.2, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_4P_pop100_keep20_full_curve.csv') df2.to_csv(output_location+'MIMIC_4P_pop100_keep20_short_curve.csv') print("Finished 4 Peaks") print("Next for 4 Peaks with 150 and 0.5") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_4P, pop_size=100, keep_pct=0.2, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_4P_pop150_keep50_full_curve.csv') df2.to_csv(output_location+'MIMIC_4P_pop150_keep50_short_curve.csv') print("Finished 4 Peaks") print("Next for 4 Peaks Big") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_4P_big, pop_size=150, keep_pct=0.2, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_4P_big_full_curve.csv') df2.to_csv(output_location+'MIMIC_4P_big_short_curve.csv') print("Finished 4 Peaks Big") print("Next for KS Small") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_KS_small, pop_size=100, keep_pct=0.5, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_KS_small_full_curve.csv') df2.to_csv(output_location+'MIMIC_KS_small_short_curve.csv') print("Finished KS small") print("Next FF small") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_FF_small, pop_size=100, keep_pct=0.5, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_FF_small_full_curve.csv') df2.to_csv(output_location+'MIMIC_KS_small_short_curve.csv') print("Finished FF Small") print("Next for 4 Peaks Small") start_time_fit = time.perf_counter() a,b,curve_output = mlrose.mimic(problem_4P_small, pop_size=150, keep_pct=0.2, max_attempts=10, max_iters=128, curve=True\ , random_state=0) end_time_fit = time.perf_counter() time_used = end_time_fit - start_time_fit df1, df2 = curve_to_df(curve_output, max) df2['time_to_128'] = time_used df1.to_csv(output_location+'MIMIC_4P_small_full_curve.csv') df2.to_csv(output_location+'MIMIC_4P_small_short_curve.csv') print("Finished 4 Peaks Small") ''' ### Now GA GA_FF = { 'pop_size': 100, #,1000], 'mutation_prob': 0.1, 'max_attempts': 30, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } GA_KS = { 'pop_size': 200, #,1000], 'mutation_prob': 0.2, 'max_attempts': 30, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } GA_4P = { 'pop_size': 200, #,1000], 'mutation_prob': 0.5, 'max_attempts': 30, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } ''' More fitness by iteration calculations #results_df, curve_output_list = fitness_by_iter('GA', problem_FF, GA_FF['max_iters'], GA_FF['random_state']\ #, pop_size=GA_FF['pop_size'], max_attempts=GA_FF['max_attempts'], curve=True, mutation_prob=GA_FF['mutation_prob']) #results_df.to_csv(output_location + 'final_GA_FF_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_FF_small, GA_FF['max_iters'], GA_FF['random_state']\ , pop_size=GA_FF['pop_size'], max_attempts=GA_FF['max_attempts'], curve=True, mutation_prob=GA_FF['mutation_prob']) results_df.to_csv(output_location + 'final_GA_FF_small_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_FF_big, GA_FF['max_iters'], GA_FF['random_state']\ , pop_size=GA_FF['pop_size'], max_attempts=GA_FF['max_attempts'], curve=True, mutation_prob=GA_FF['mutation_prob']) results_df.to_csv(output_location + 'final_GA_FF_big_attempt_8am.csv') #results_df, curve_output_list = fitness_by_iter('GA', problem_4P, GA_4P['max_iters'], GA_4P['random_state']\ #, pop_size=GA_4P['pop_size'], max_attempts=GA_4P['max_attempts'], curve=True, mutation_prob=GA_4P['mutation_prob']) #results_df.to_csv(output_location + 'final_GA_4P_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_4P_big, GA_4P['max_iters'], GA_4P['random_state']\ , pop_size=GA_4P['pop_size'], max_attempts=GA_4P['max_attempts'], curve=True, mutation_prob=GA_4P['mutation_prob']) results_df.to_csv(output_location + 'final_GA_4P_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_4P_small, GA_4P['max_iters'], GA_4P['random_state']\ , pop_size=GA_4P['pop_size'], max_attempts=GA_4P['max_attempts'], curve=True, mutation_prob=GA_4P['mutation_prob']) results_df.to_csv(output_location + 'final_GA_4P_small_attempt_8am.csv') #results_df, curve_output_list = fitness_by_iter('GA', problem_KS, GA_KS['max_iters'], GA_KS['random_state']\ #, pop_size=GA_KS['pop_size'], max_attempts=GA_KS['max_attempts'], curve=True, mutation_prob=GA_KS['mutation_prob']) #results_df.to_csv(output_location + 'final_GA_KS_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_KS_big, GA_KS['max_iters'], GA_KS['random_state']\ , pop_size=GA_KS['pop_size'], max_attempts=GA_KS['max_attempts'], curve=True, mutation_prob=GA_KS['mutation_prob']) results_df.to_csv(output_location + 'final_GA_KS_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_KS_small, GA_KS['max_iters'], GA_KS['random_state']\ , pop_size=GA_KS['pop_size'], max_attempts=GA_KS['max_attempts'], curve=True, mutation_prob=GA_KS['mutation_prob']) results_df.to_csv(output_location + 'final_GA_KS_small_attempt_8am.csv') ''' ########### SA print("now doing SA") SA_4P = { 'max_attempts': 10, 'schedule': mlrose.GeomDecay(init_temp=100, decay=0.8), 'max_iters': max_iters_list_full, 'random_state': rand_list_full } SA_FF = { 'max_attempts': 10, 'schedule': mlrose.GeomDecay(init_temp=100, decay=0.8), 'max_iters': max_iters_list_full, 'random_state': rand_list_full } results_df, curve_output_list = fitness_by_iter('SA', problem_4P, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_4P_big, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_4P, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_small_attempt_8am.csv') ''' more fitness by iteration calculations #results_df, curve_output_list = fitness_by_iter('SA', problem_FF, SA_FF['max_iters'], SA_FF['random_state']\ #, schedule=SA_FF['schedule'], max_attempts=SA_FF['max_attempts'], curve=True) #results_df.to_csv(output_location + 'final_SA_FF_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_FF_big, SA_FF['max_iters'], SA_FF['random_state']\ , schedule=SA_FF['schedule'], max_attempts=SA_FF['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_FF_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_FF_small, SA_FF['max_iters'], SA_FF['random_state']\ , schedule=SA_FF['schedule'], max_attempts=SA_FF['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_FF_small_attempt_8am.csv') SA_4P = { 'max_attempts':10, 'schedule':mlrose.GeomDecay(init_temp=100, decay=0.8), 'max_iters':max_iters_list_full, 'random_state':rand_list_full } results_df, curve_output_list = fitness_by_iter('KS', problem_4P, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_4P_big, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_4P, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_small_attempt_8am.csv') ''' print("picking up where I left off on making the final curves..") SA_KS = { 'max_attempts': 10, 'schedule': mlrose.GeomDecay(init_temp=1000, decay=0.99), 'max_iters': max_iters_list_full, 'random_state': rand_list_full } ''' more fitness by iteration calculations results_df, curve_output_list = fitness_by_iter('SA', problem_KS, SA_KS['max_iters'], SA_KS['random_state']\ , schedule=SA_KS['schedule'], max_attempts=SA_KS['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_KS_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_KS_big, SA_KS['max_iters'], SA_KS['random_state']\ , schedule=SA_KS['schedule'], max_attempts=SA_KS['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_KS_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_KS_small, SA_KS['max_iters'], SA_KS['random_state']\ , schedule=SA_KS['schedule'], max_attempts=SA_KS['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_KS_small_attempt_8am.csv') ''' RHC_KS = { 'max_attempts': 50, 'restarts': 20, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } ''' results_df, curve_output_list = fitness_by_iter('RHC', problem_KS, RHC_KS['max_iters'], RHC_KS['random_state']\ , restarts=RHC_KS['restarts'], max_attempts=RHC_KS['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_KS_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('RHC', problem_KS_big, RHC_KS['max_iters'], RHC_KS['random_state']\ , restarts=RHC_KS['restarts'], max_attempts=RHC_KS['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_KS_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('RHC', problem_KS_small, RHC_KS['max_iters'], RHC_KS['random_state']\ , restarts=RHC_KS['restarts'], max_attempts=RHC_KS['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_KS_small_attempt_8am.csv') ''' RHC_FF = { 'max_attempts': 50, 'restarts': 20, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } ''' results_df, curve_output_list = fitness_by_iter('RHC', problem_FF, RHC_FF['max_iters'], RHC_FF['random_state']\ , restarts=RHC_FF['restarts'], max_attempts=RHC_FF['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_FF_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('RHC', problem_FF_big, RHC_FF['max_iters'], RHC_FF['random_state']\ , restarts=RHC_FF['restarts'], max_attempts=RHC_FF['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_FF_big_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('RHC', problem_FF_small, RHC_FF['max_iters'], RHC_FF['random_state']\ , restarts=RHC_FF['restarts'], max_attempts=RHC_FF['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_FF_small_attempt_8am.csv') ''' RHC_4P = { 'max_attempts': 50, 'restarts': 20, 'max_iters': max_iters_list_full, 'random_state': rand_list_full } ''' results_df, curve_output_list = fitness_by_iter('RHC', problem_4P, RHC_4P['max_iters'], RHC_4P['random_state']\ , restarts=RHC_4P['restarts'], max_attempts=RHC_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_4P_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('RHC', problem_4P_small, RHC_4P['max_iters'], RHC_4P['random_state']\ , restarts=RHC_4P['restarts'], max_attempts=RHC_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_4P_small_attempt_8am.csv') results_df, curve_output_list = fitness_by_iter('RHC', problem_4P_big, RHC_4P['max_iters'], RHC_4P['random_state']\ , restarts=RHC_4P['restarts'], max_attempts=RHC_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_RHS_4P_big_attempt_8am.csv') ''' ## where it stopped print("I will now make the complexity curves for other algos") SA_4P_hacked = { 'max_attempts': 10, 'schedule': mlrose.GeomDecay(init_temp=100, decay=0.99), 'max_iters': max_iters_list_full, 'random_state': rand_list_full } ''' results_df, curve_output_list = fitness_by_iter('SA', problem_4P, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=SA_4P_hacked['schedule'], max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_decay_99.csv') results_df, curve_output_list = fitness_by_iter('SA', problem_4P, SA_4P['max_iters'], SA_4P['random_state']\ , schedule=mlrose.GeomDecay(init_temp=1, decay=0.8), max_attempts=SA_4P['max_attempts'], curve=True) results_df.to_csv(output_location + 'final_SA_4P_T_1_decay_80.csv') results_df, curve_output_list = fitness_by_iter('GA', problem_KS, GA_KS['max_iters'], GA_KS['random_state']\ , pop_size=GA_KS['pop_size'], max_attempts=GA_KS['max_attempts'], curve=True, mutation_prob=0.1) results_df.to_csv(output_location + 'final_GA_KS_mutation_01.csv') ''' results_df, curve_output_list = fitness_by_iter('GA', problem_KS, GA_KS['max_iters'], GA_KS['random_state']\ , pop_size=100, max_attempts=GA_KS['max_attempts'], curve=True, mutation_prob=0.2) results_df.to_csv(output_location + 'final_GA_KS_mutation_02_pop_100.csv') ## Need a few more MIMIC chart inputs #print("Need a few more MIMIC chart inputs, so I will now make those") #print("Next FF p=100 keep=0.2") ''' MIMIC inputs for charts
attempts = max_it * 3/4 rhc_fitnesses = [] sa_fitnesses = [] ga_fitnesses = [] m_fitnesses = [] rhc_times = [] sa_times = [] ga_times = [] m_times = [] test_range = [4,6,8,10,15,20,25] final_prob_len = 30 fitness_cust = mlr.SixPeaks(t_pct=0.1) schedule = mlr.GeomDecay(decay=0.99) print(f"Attempts: {attempts}") print(f"Max Iterations: {max_it}") print(f"Problem Sizes: {test_range}") print(f"Last Problem Size: {final_prob_len}\n\n") part2_time = 0.0 print(f"\n######### PART 2 #########\n") for i in test_range: start = time.time() print(f"Running for subproblem size: {i}") init = np.random.choice([0,1], size=i, replace=True)
learning_rate=1.4e-4, max_iters=MAX_ITERS, clip_max=1e3), Algorithm('RHC', 'random_hill_climb', rhc_grid, max_iters=MAX_ITERS, max_attempts=20, restarts=10, learning_rate=2.5, clip_max=1e3), Algorithm('SA', 'simulated_annealing', simulated_annealing_grid, learning_rate=2, schedule=mlrose.GeomDecay(), max_attempts=30, max_iters=MAX_ITERS, clip_max=1e3, init_temp=1, decay=1 - 0.01, min_temp=3.36e-5), Algorithm('GA', 'genetic_alg', genetic_grid, max_iters=400, mutation_prob=0.1, learning_rate=0.01, max_attempts=10, clip_max=0.7, pop_size=200),
from optimization_objects.six_peaks import get_six_peaks from visualization.plot_graphs import plot_sim_ann_schedules input_sizes = [25, 50, 75, 100, 150, 200, 300] opt_probs = [ (get_one_max, 'one max'), (get_four_peaks, 'four peaks'), (get_six_peaks, 'six peaks'), (get_continuous_peaks, 'continuous peaks'), (get_flip_flop, 'flip flop'), ] sim_ann = get_sim_ann options = [(mlrose.GeomDecay(), 'geometric'), (mlrose.ArithDecay(), 'arithmetic'), (mlrose.ExpDecay(), 'exponential')] metric_dict = {prob[1]: {} for prob in opt_probs} for size in input_sizes: print('####################################### New input size ', size, ' ##############################################') for prob in opt_probs: problem_name = prob[1] problem = prob[0](size) print('----------------------------------New Problem ', problem_name, ' ------------------------------------------') for opt in options: option_name = opt[1]