def get_param_grids(): lrs = [10**i for i in range(-4, -2)] gd_params = {'learning_rate': [0.0001], 'max_iters': [2000]} rhc_params = { 'learning_rate': [0.1], 'restarts': [100], #[10] 'max_iters': [2000] } sa_params = { 'learning_rate': [0.0001, 0.001, 0.01, 0.1], 'schedule': [ ExpDecay(), ExpDecay(exp_const=0.002), ExpDecay(exp_const=1 / 2000), GeomDecay(), GeomDecay(decay=.999), ArithDecay(), ArithDecay(decay=1 / 2000), ], 'max_iters': [2000] } ga_params = { 'learning_rate': [0.0001, 0.001, 0.01, 0.1], 'pop_size': [400], #[200, 400], #[200] 'mutation_prob': [0.1, .2, .5, .9], 'max_iters': [500] } params = { 'gradient_descent': gd_params, 'random_hill_climb': rhc_params, 'simulated_annealing': sa_params, 'genetic_alg': ga_params } return params
def test_geom_below_min(): """Test geometric decay evaluation function for case where result is below the minimum""" schedule = GeomDecay(init_temp=10, decay=0.95, min_temp=1) x = schedule.evaluate(50) assert x == 1
def test_geom_above_min(): """Test geometric decay evaluation function for case where result is above the minimum""" schedule = GeomDecay(init_temp=10, decay=0.95, min_temp=1) x = schedule.evaluate(5) assert round(x, 5) == 7.73781
def get_param_grids(): lrs = [10**i for i in range(-4, -2)] gd_params = {'learning_rate': lrs, 'max_iters': [2000]} rhc_params = { 'learning_rate': lrs, 'restarts': [10, 30], #[10] 'max_iters': [2000] } sa_params = { 'learning_rate': lrs, 'schedule': [GeomDecay(), ArithDecay(), ExpDecay()], 'max_iters': [2000] } ga_params = { 'learning_rate': lrs, 'pop_size': [200], #[200] 'mutation_prob': [0.05, 0.1, 0.2], #[0.1] 'max_iters': [500] } params = { 'gradient_descent': gd_params, 'random_hill_climb': rhc_params, 'simulated_annealing': sa_params, 'genetic_alg': ga_params } return params
def __init__(self, hidden_nodes=None, activation='relu', algorithm='random_hill_climb', max_iters=100, bias=True, is_classifier=True, learning_rate=0.1, early_stopping=False, clip_max=1e+10, restarts=0, schedule=GeomDecay(), pop_size=200, mutation_prob=0.1, max_attempts=10, random_state=None, curve=False): super().__init__() if hidden_nodes is None: self.hidden_nodes = [] else: self.hidden_nodes = hidden_nodes self.activation_dict = { 'identity': identity, 'relu': relu, 'sigmoid': sigmoid, 'tanh': tanh } self.activation = activation self.algorithm = algorithm self.max_iters = max_iters self.bias = bias self.is_classifier = is_classifier self.learning_rate = learning_rate self.early_stopping = early_stopping self.clip_max = clip_max self.restarts = restarts self.schedule = schedule self.pop_size = pop_size self.mutation_prob = mutation_prob self.max_attempts = max_attempts self.random_state = random_state self.curve = curve self.node_list = [] self.fitted_weights = [] self.loss = np.inf self.output_activation = None self.predicted_probs = [] self.fitness_curve = []
def run_experiment(experiment_name, model, init_temp=15.0, max_it=2000, title="Foo", analyze=True): model.schedule = GeomDecay(init_temp=init_temp) model.max_iters = max_it run_start = time.perf_counter() if analyze: result = model.fit(X_train, y_train) run_end = time.perf_counter() print(f'Run time: {run_end - run_start}') run_time = run_end - run_start curves_df = pd.DataFrame(columns=['Fitness', 'Time', 'Restarts']) curves_df['Fitness'] = model.fitness_curve curves_df['Time'] = run_time curves_df['Restarts'] = restarts curves_df['Iteration'] = curves_df.index # Predict labels for train set and assess accuracy y_train_pred = model.predict(X_train) y_train_accuracy = accuracy_score(y_train, y_train_pred) print("RHC NN train accuracy") print(y_train_accuracy) curves_df['TrainAcc'] = y_train_accuracy # Predict labels for test set and assess accuracy y_test_pred = model.predict(X_test) y_test_accuracy = accuracy_score(y_test, y_test_pred) curves_df['TestAcc'] = y_test_accuracy print("RHC NN test accuracy") print(y_test_accuracy) dump_df_to_disk(curves_df, 'sa', 'curves_df', experiment_name, 'sa') filename = 'sa/' + experiment_name + '/sa__' + experiment_name + '__curves_df.csv' data = pd.read_csv(filename, header=0) plot_curve_sa(data, title)
def SimulatedAnnealing(self): start = time.time() KSbest_state, KSbest_fitness, KScurve = simulated_annealing( self.knapsack, curve=True) saKST = time.time() - start start = time.time() FPbest_state, FPbest_fitness, FPcurve = simulated_annealing( self.fourpeaks, curve=True) saFPT = time.time() - start start = time.time() CObest_state, CObest_fitness, COcurve = simulated_annealing( self.countones, schedule=GeomDecay(init_temp=.01, decay=.99), curve=True) saCOT = time.time() - start return KSbest_fitness, FPbest_fitness, CObest_fitness, saKST, saFPT, saCOT, KScurve, FPcurve, COcurve
def get_nn_error(x_train_scaled, x_test_scaled, y_train, y_test, iter, param_name, param_value, algorithm): # Initialize neural network object and fit object start_time = time.time() # Random Hill Climb if param_name == "": nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5], activation='relu', algorithm=algorithm, max_iters=iter, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=5, max_attempts=100, random_state=3) # Simulated Annealing elif param_name == "init_temp": nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5], activation='relu', algorithm='simulated_annealing', max_iters=iter, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=5, max_attempts=100, random_state=3, schedule=GeomDecay(init_temp=param_value)) # Simulated Annealing elif param_name == "decay": nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5], activation='relu', algorithm='simulated_annealing', max_iters=iter, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=5, max_attempts=100, random_state=3, schedule=GeomDecay(init_temp=param_value)) elif param_name == "pop_size": nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5], activation='relu', algorithm='genetic_alg', max_iters=iter, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=5, max_attempts=100, pop_size=param_value, random_state=3) elif param_name == "mutation_prob": nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5], activation='relu', algorithm='genetic_alg', max_iters=iter, bias=True, is_classifier=True, learning_rate=0.0001, early_stopping=True, clip_max=5, max_attempts=100, mutation_prob=param_value, random_state=3) nn_ro.fit(x_train_scaled, y_train) # get running time end_time = time.time() train_time = end_time - start_time # get prediction accuracy y_pred = nn_ro.predict(x_test_scaled) test_accuracy = accuracy_score(y_test, y_pred) # Get Mean Squared Error MSE = mean_squared_error(y_test, y_pred) return test_accuracy, MSE, train_time
def get_optimizers(): return { 'RHC': { 'function': random_hill_climb, 'kwargs': [{}], }, 'SA': { 'function': simulated_annealing, 'kwargs': [ { 'schedule': ExpDecay() }, { 'schedule': ExpDecay(exp_const=0.01) }, { 'schedule': ExpDecay(exp_const=0.002) }, { 'schedule': GeomDecay() }, { 'schedule': GeomDecay(decay=.98) }, { 'schedule': GeomDecay(decay=.96) }, { 'schedule': ArithDecay() }, { 'schedule': ArithDecay(decay=0.001) }, { 'schedule': ArithDecay(decay=1 / 5000) }, { 'schedule': ExpDecay(exp_const=0.002) }, ] }, 'GA': { 'function': genetic_alg, 'kwargs': [ { 'pop_size': 200, 'mutation_prob': 0.1 }, { 'pop_size': 200, 'mutation_prob': 0.2 }, { 'pop_size': 200, 'mutation_prob': 0.05 }, { 'pop_size': 800, 'mutation_prob': 0.1 }, ] }, 'MIMIC': { 'function': mimic, 'kwargs': [ { 'pop_size': 200, 'keep_pct': 0.2, #'fast_mimic': True }, { 'pop_size': 200, 'keep_pct': 0.4, #'fast_mimic': True }, { 'pop_size': 400, 'keep_pct': 0.2, }, ] } }
def get_optimizers(): return { #'RHC': { # 'function': random_hill_climb, # 'kwargs': [ # {} # ], #}, 'SA': { 'function': simulated_annealing, 'kwargs': [ { 'schedule': ExpDecay() }, #{'schedule': ExpDecay(exp_const=0.01)}, { 'schedule': ExpDecay(exp_const=0.002) }, { 'schedule': ExpDecay(exp_const=1 / 5000) }, { 'schedule': GeomDecay() }, #{'schedule': GeomDecay(decay=.98)}, { 'schedule': GeomDecay(decay=.9993) }, { 'schedule': ArithDecay() }, #{'schedule': ArithDecay(decay=0.001)}, { 'schedule': ArithDecay(decay=1 / 5000) }, ] }, #'GA': { # 'function': genetic_alg, # 'kwargs': [ # #{'pop_size': 200, # # 'mutation_prob': 0.1 # #}, # #{'pop_size': 200, # # 'mutation_prob': 0.2 # #}, # #{'pop_size': 200, # # 'mutation_prob': 0.05 # #}, # #{'pop_size': 400, # # 'mutation_prob': 0.4 # #}, # #{'pop_size': 400, # # 'mutation_prob': 0.2 # #}, # #{'pop_size': 400, # # 'mutation_prob': 0.1 # #}, # {'pop_size': 800, # 'mutation_prob': 0.1 # }, # ] #}, #'MIMIC': { # 'function': mimic, # 'kwargs': [ # {'pop_size': 400, # 'keep_pct': 0.2, # }, # {'pop_size': 400, # 'keep_pct': 0.1, # }, # {'pop_size': 400, # 'keep_pct': 0.05, # }, # #{'pop_size': 800, # # 'keep_pct': 0.2, # #}, # #{'pop_size': 800, # # 'keep_pct': 0.1, # #}, # ] #} }
def sa(): def plot_curve_sa(df, title): iterations = df['Iteration'] fitness1 = df['Fitness'] time_val = df['Time'].values[0] train_acc = df['TrainAcc'].values[0] test_acc = df['TestAcc'].values[0] fig, ax1 = plt.subplots() color = 'tab:red' color2 = 'tab:blue' ax1.set_xlabel('iterations') ax1.set_ylabel('fitness', color=color) ax1.plot(iterations, 1 / fitness1, color=color, label='fitness') ax1.tick_params(axis='y', labelcolor=color) ax1.text(0.65, 0.35, 'Time:' + str(round(time_val, 3)), transform=ax1.transAxes, color=color) ax1.text(0.65, 0.30, 'Train Acc:' + str(round(train_acc, 3)), transform=ax1.transAxes, color=color) ax1.text(0.65, 0.25, 'Test Acc:' + str(round(test_acc, 3)), transform=ax1.transAxes, color=color) # Create plot plt.title(title) plt.legend(loc="best") plt.tight_layout() plt.show() def run_experiment(experiment_name, model, init_temp=15.0, max_it=2000, title="Foo", analyze=True): model.schedule = GeomDecay(init_temp=init_temp) model.max_iters = max_it run_start = time.perf_counter() if analyze: result = model.fit(X_train, y_train) run_end = time.perf_counter() print(f'Run time: {run_end - run_start}') run_time = run_end - run_start curves_df = pd.DataFrame(columns=['Fitness', 'Time', 'Restarts']) curves_df['Fitness'] = model.fitness_curve curves_df['Time'] = run_time curves_df['Restarts'] = restarts curves_df['Iteration'] = curves_df.index # Predict labels for train set and assess accuracy y_train_pred = model.predict(X_train) y_train_accuracy = accuracy_score(y_train, y_train_pred) print("RHC NN train accuracy") print(y_train_accuracy) curves_df['TrainAcc'] = y_train_accuracy # Predict labels for test set and assess accuracy y_test_pred = model.predict(X_test) y_test_accuracy = accuracy_score(y_test, y_test_pred) curves_df['TestAcc'] = y_test_accuracy print("RHC NN test accuracy") print(y_test_accuracy) dump_df_to_disk(curves_df, 'sa', 'curves_df', experiment_name, 'sa') filename = 'sa/' + experiment_name + '/sa__' + experiment_name + '__curves_df.csv' data = pd.read_csv(filename, header=0) plot_curve_sa(data, title) experiment_name = 'nn_sa' restarts = 1 model = NeuralNetwork(hidden_nodes=[70], activation='relu', \ algorithm='simulated_annealing', max_iters=2000, \ bias=True, is_classifier=True, learning_rate=0.15, \ early_stopping=True, clip_max=5, max_attempts=100, \ random_state=SEED, schedule=GeomDecay(init_temp=15.0), curve=True) #run_experiment('nn_sa_t15', model, 15) #run_experiment('nn_sa_t15', model, 30) run_experiment('nn_sa_t50', model, 50, title='Fitness Curve NN(70) - SA T=50', analyze=False) #run_experiment('nn_rhc_r80', model, 80) #run_experiment('nn_rhc_r130_it4000', model, 130, 4000) run_experiment('nn_rhc_r130_it5000', model, 130, 5000, title='Fitness Curve NN(70) - SA T=130', analyze=False)