def get_param_grids():
lrs = [10**i for i in range(-4, -2)]
gd_params = {'learning_rate': [0.0001], 'max_iters': [2000]}
rhc_params = {
'learning_rate': [0.1],
'restarts': [100], #[10]
'max_iters': [2000]
}
sa_params = {
'learning_rate': [0.0001, 0.001, 0.01, 0.1],
'schedule': [
ExpDecay(),
ExpDecay(exp_const=0.002),
ExpDecay(exp_const=1 / 2000),
GeomDecay(),
GeomDecay(decay=.999),
ArithDecay(),
ArithDecay(decay=1 / 2000),
],
'max_iters': [2000]
}
ga_params = {
'learning_rate': [0.0001, 0.001, 0.01, 0.1],
'pop_size': [400], #[200, 400], #[200]
'mutation_prob': [0.1, .2, .5, .9],
'max_iters': [500]
}
params = {
'gradient_descent': gd_params,
'random_hill_climb': rhc_params,
'simulated_annealing': sa_params,
'genetic_alg': ga_params
}
return params
def test_geom_below_min():
"""Test geometric decay evaluation function for case where result is
below the minimum"""
schedule = GeomDecay(init_temp=10, decay=0.95, min_temp=1)
x = schedule.evaluate(50)
assert x == 1
def test_geom_above_min():
"""Test geometric decay evaluation function for case where result is
above the minimum"""
schedule = GeomDecay(init_temp=10, decay=0.95, min_temp=1)
x = schedule.evaluate(5)
assert round(x, 5) == 7.73781
def get_param_grids():
lrs = [10**i for i in range(-4, -2)]
gd_params = {'learning_rate': lrs, 'max_iters': [2000]}
rhc_params = {
'learning_rate': lrs,
'restarts': [10, 30], #[10]
'max_iters': [2000]
}
sa_params = {
'learning_rate': lrs,
'schedule': [GeomDecay(), ArithDecay(),
ExpDecay()],
'max_iters': [2000]
}
ga_params = {
'learning_rate': lrs,
'pop_size': [200], #[200]
'mutation_prob': [0.05, 0.1, 0.2], #[0.1]
'max_iters': [500]
}
params = {
'gradient_descent': gd_params,
'random_hill_climb': rhc_params,
'simulated_annealing': sa_params,
'genetic_alg': ga_params
}
return params
def __init__(self,
hidden_nodes=None,
activation='relu',
algorithm='random_hill_climb',
max_iters=100,
bias=True,
is_classifier=True,
learning_rate=0.1,
early_stopping=False,
clip_max=1e+10,
restarts=0,
schedule=GeomDecay(),
pop_size=200,
mutation_prob=0.1,
max_attempts=10,
random_state=None,
curve=False):
super().__init__()
if hidden_nodes is None:
self.hidden_nodes = []
else:
self.hidden_nodes = hidden_nodes
self.activation_dict = {
'identity': identity,
'relu': relu,
'sigmoid': sigmoid,
'tanh': tanh
}
self.activation = activation
self.algorithm = algorithm
self.max_iters = max_iters
self.bias = bias
self.is_classifier = is_classifier
self.learning_rate = learning_rate
self.early_stopping = early_stopping
self.clip_max = clip_max
self.restarts = restarts
self.schedule = schedule
self.pop_size = pop_size
self.mutation_prob = mutation_prob
self.max_attempts = max_attempts
self.random_state = random_state
self.curve = curve
self.node_list = []
self.fitted_weights = []
self.loss = np.inf
self.output_activation = None
self.predicted_probs = []
self.fitness_curve = []
def run_experiment(experiment_name,
model,
init_temp=15.0,
max_it=2000,
title="Foo",
analyze=True):
model.schedule = GeomDecay(init_temp=init_temp)
model.max_iters = max_it
run_start = time.perf_counter()
if analyze:
result = model.fit(X_train, y_train)
run_end = time.perf_counter()
print(f'Run time: {run_end - run_start}')
run_time = run_end - run_start
curves_df = pd.DataFrame(columns=['Fitness', 'Time', 'Restarts'])
curves_df['Fitness'] = model.fitness_curve
curves_df['Time'] = run_time
curves_df['Restarts'] = restarts
curves_df['Iteration'] = curves_df.index
# Predict labels for train set and assess accuracy
y_train_pred = model.predict(X_train)
y_train_accuracy = accuracy_score(y_train, y_train_pred)
print("RHC NN train accuracy")
print(y_train_accuracy)
curves_df['TrainAcc'] = y_train_accuracy
# Predict labels for test set and assess accuracy
y_test_pred = model.predict(X_test)
y_test_accuracy = accuracy_score(y_test, y_test_pred)
curves_df['TestAcc'] = y_test_accuracy
print("RHC NN test accuracy")
print(y_test_accuracy)
dump_df_to_disk(curves_df, 'sa', 'curves_df', experiment_name,
'sa')
filename = 'sa/' + experiment_name + '/sa__' + experiment_name + '__curves_df.csv'
data = pd.read_csv(filename, header=0)
plot_curve_sa(data, title)
def SimulatedAnnealing(self):
start = time.time()
KSbest_state, KSbest_fitness, KScurve = simulated_annealing(
self.knapsack, curve=True)
saKST = time.time() - start
start = time.time()
FPbest_state, FPbest_fitness, FPcurve = simulated_annealing(
self.fourpeaks, curve=True)
saFPT = time.time() - start
start = time.time()
CObest_state, CObest_fitness, COcurve = simulated_annealing(
self.countones,
schedule=GeomDecay(init_temp=.01, decay=.99),
curve=True)
saCOT = time.time() - start
return KSbest_fitness, FPbest_fitness, CObest_fitness, saKST, saFPT, saCOT, KScurve, FPcurve, COcurve
def get_nn_error(x_train_scaled, x_test_scaled, y_train, y_test, iter,
param_name, param_value, algorithm):
# Initialize neural network object and fit object
start_time = time.time()
# Random Hill Climb
if param_name == "":
nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5],
activation='relu',
algorithm=algorithm,
max_iters=iter,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=5,
max_attempts=100,
random_state=3)
# Simulated Annealing
elif param_name == "init_temp":
nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5],
activation='relu',
algorithm='simulated_annealing',
max_iters=iter,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=5,
max_attempts=100,
random_state=3,
schedule=GeomDecay(init_temp=param_value))
# Simulated Annealing
elif param_name == "decay":
nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5],
activation='relu',
algorithm='simulated_annealing',
max_iters=iter,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=5,
max_attempts=100,
random_state=3,
schedule=GeomDecay(init_temp=param_value))
elif param_name == "pop_size":
nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5],
activation='relu',
algorithm='genetic_alg',
max_iters=iter,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=5,
max_attempts=100,
pop_size=param_value,
random_state=3)
elif param_name == "mutation_prob":
nn_ro = mlrose.NeuralNetwork(hidden_nodes=[5],
activation='relu',
algorithm='genetic_alg',
max_iters=iter,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=5,
max_attempts=100,
mutation_prob=param_value,
random_state=3)
nn_ro.fit(x_train_scaled, y_train)
# get running time
end_time = time.time()
train_time = end_time - start_time
# get prediction accuracy
y_pred = nn_ro.predict(x_test_scaled)
test_accuracy = accuracy_score(y_test, y_pred)
# Get Mean Squared Error
MSE = mean_squared_error(y_test, y_pred)
return test_accuracy, MSE, train_time
def get_optimizers():
return {
'RHC': {
'function': random_hill_climb,
'kwargs': [{}],
},
'SA': {
'function':
simulated_annealing,
'kwargs': [
{
'schedule': ExpDecay()
},
{
'schedule': ExpDecay(exp_const=0.01)
},
{
'schedule': ExpDecay(exp_const=0.002)
},
{
'schedule': GeomDecay()
},
{
'schedule': GeomDecay(decay=.98)
},
{
'schedule': GeomDecay(decay=.96)
},
{
'schedule': ArithDecay()
},
{
'schedule': ArithDecay(decay=0.001)
},
{
'schedule': ArithDecay(decay=1 / 5000)
},
{
'schedule': ExpDecay(exp_const=0.002)
},
]
},
'GA': {
'function':
genetic_alg,
'kwargs': [
{
'pop_size': 200,
'mutation_prob': 0.1
},
{
'pop_size': 200,
'mutation_prob': 0.2
},
{
'pop_size': 200,
'mutation_prob': 0.05
},
{
'pop_size': 800,
'mutation_prob': 0.1
},
]
},
'MIMIC': {
'function':
mimic,
'kwargs': [
{
'pop_size': 200,
'keep_pct': 0.2,
#'fast_mimic': True
},
{
'pop_size': 200,
'keep_pct': 0.4,
#'fast_mimic': True
},
{
'pop_size': 400,
'keep_pct': 0.2,
},
]
}
}
def get_optimizers():
return {
#'RHC': {
# 'function': random_hill_climb,
# 'kwargs': [
# {}
# ],
#},
'SA': {
'function':
simulated_annealing,
'kwargs': [
{
'schedule': ExpDecay()
},
#{'schedule': ExpDecay(exp_const=0.01)},
{
'schedule': ExpDecay(exp_const=0.002)
},
{
'schedule': ExpDecay(exp_const=1 / 5000)
},
{
'schedule': GeomDecay()
},
#{'schedule': GeomDecay(decay=.98)},
{
'schedule': GeomDecay(decay=.9993)
},
{
'schedule': ArithDecay()
},
#{'schedule': ArithDecay(decay=0.001)},
{
'schedule': ArithDecay(decay=1 / 5000)
},
]
},
#'GA': {
# 'function': genetic_alg,
# 'kwargs': [
# #{'pop_size': 200,
# # 'mutation_prob': 0.1
# #},
# #{'pop_size': 200,
# # 'mutation_prob': 0.2
# #},
# #{'pop_size': 200,
# # 'mutation_prob': 0.05
# #},
# #{'pop_size': 400,
# # 'mutation_prob': 0.4
# #},
# #{'pop_size': 400,
# # 'mutation_prob': 0.2
# #},
# #{'pop_size': 400,
# # 'mutation_prob': 0.1
# #},
# {'pop_size': 800,
# 'mutation_prob': 0.1
# },
# ]
#},
#'MIMIC': {
# 'function': mimic,
# 'kwargs': [
# {'pop_size': 400,
# 'keep_pct': 0.2,
# },
# {'pop_size': 400,
# 'keep_pct': 0.1,
# },
# {'pop_size': 400,
# 'keep_pct': 0.05,
# },
# #{'pop_size': 800,
# # 'keep_pct': 0.2,
# #},
# #{'pop_size': 800,
# # 'keep_pct': 0.1,
# #},
# ]
#}
}
def sa():
def plot_curve_sa(df, title):
iterations = df['Iteration']
fitness1 = df['Fitness']
time_val = df['Time'].values[0]
train_acc = df['TrainAcc'].values[0]
test_acc = df['TestAcc'].values[0]
fig, ax1 = plt.subplots()
color = 'tab:red'
color2 = 'tab:blue'
ax1.set_xlabel('iterations')
ax1.set_ylabel('fitness', color=color)
ax1.plot(iterations, 1 / fitness1, color=color, label='fitness')
ax1.tick_params(axis='y', labelcolor=color)
ax1.text(0.65,
0.35,
'Time:' + str(round(time_val, 3)),
transform=ax1.transAxes,
color=color)
ax1.text(0.65,
0.30,
'Train Acc:' + str(round(train_acc, 3)),
transform=ax1.transAxes,
color=color)
ax1.text(0.65,
0.25,
'Test Acc:' + str(round(test_acc, 3)),
transform=ax1.transAxes,
color=color)
# Create plot
plt.title(title)
plt.legend(loc="best")
plt.tight_layout()
plt.show()
def run_experiment(experiment_name,
model,
init_temp=15.0,
max_it=2000,
title="Foo",
analyze=True):
model.schedule = GeomDecay(init_temp=init_temp)
model.max_iters = max_it
run_start = time.perf_counter()
if analyze:
result = model.fit(X_train, y_train)
run_end = time.perf_counter()
print(f'Run time: {run_end - run_start}')
run_time = run_end - run_start
curves_df = pd.DataFrame(columns=['Fitness', 'Time', 'Restarts'])
curves_df['Fitness'] = model.fitness_curve
curves_df['Time'] = run_time
curves_df['Restarts'] = restarts
curves_df['Iteration'] = curves_df.index
# Predict labels for train set and assess accuracy
y_train_pred = model.predict(X_train)
y_train_accuracy = accuracy_score(y_train, y_train_pred)
print("RHC NN train accuracy")
print(y_train_accuracy)
curves_df['TrainAcc'] = y_train_accuracy
# Predict labels for test set and assess accuracy
y_test_pred = model.predict(X_test)
y_test_accuracy = accuracy_score(y_test, y_test_pred)
curves_df['TestAcc'] = y_test_accuracy
print("RHC NN test accuracy")
print(y_test_accuracy)
dump_df_to_disk(curves_df, 'sa', 'curves_df', experiment_name,
'sa')
filename = 'sa/' + experiment_name + '/sa__' + experiment_name + '__curves_df.csv'
data = pd.read_csv(filename, header=0)
plot_curve_sa(data, title)
experiment_name = 'nn_sa'
restarts = 1
model = NeuralNetwork(hidden_nodes=[70], activation='relu', \
algorithm='simulated_annealing', max_iters=2000, \
bias=True, is_classifier=True, learning_rate=0.15, \
early_stopping=True, clip_max=5, max_attempts=100, \
random_state=SEED,
schedule=GeomDecay(init_temp=15.0),
curve=True)
#run_experiment('nn_sa_t15', model, 15)
#run_experiment('nn_sa_t15', model, 30)
run_experiment('nn_sa_t50',
model,
50,
title='Fitness Curve NN(70) - SA T=50',
analyze=False)
#run_experiment('nn_rhc_r80', model, 80)
#run_experiment('nn_rhc_r130_it4000', model, 130, 4000)
run_experiment('nn_rhc_r130_it5000',
model,
130,
5000,
title='Fitness Curve NN(70) - SA T=130',
analyze=False)