def calcTestScore(self):
legendCounter = 0
learners = []
legend = []
nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='gradient_descent',
max_iters=200, bias=True, is_classifier=True, learning_rate=0.001,
early_stopping=False, clip_max=1e10, curve=False)
learners.append(nn)
legend.append('GD')
nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='simulated_annealing',
max_iters=800, bias=True, is_classifier=True, learning_rate=0.001,
early_stopping=False, clip_max=1e10, schedule=mlrose.GeomDecay(),
curve=False)
learners.append(nn)
legend.append('SA')
nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='random_hill_climb',
max_iters=600, bias=True, is_classifier=True, learning_rate=0.001,
early_stopping=False, clip_max=1e10, curve=False)
learners.append(nn)
legend.append('RHC')
nn = mlrose.NeuralNetwork(hidden_nodes=[20], activation='relu', algorithm='genetic_alg',
max_iters=200, bias=True, is_classifier=True, learning_rate=0.001,
early_stopping=False, clip_max=1e10, pop_size=300, mutation_prob=0.2,
curve=False)
learners.append(nn)
legend.append('GA')
for learner in learners:
timeStart = time.time()
learner.fit(self.dataset1.training_x, self.dataset1.training_y)
testingF1 = f1_score(learner.predict(self.dataset1.testing_x), self.dataset1.testing_y, average='weighted')
print('Testing F1-Score for' + legend[legendCounter] + ': ' + str(testingF1))
legendCounter += 1
def predict(gradient_descent, cv, iter, restart, X_train, X_test, y_train, y_test):
y_train_accuracy = 0
y_test_accuracy = 0
for i in range(restart):
# Standardlization
scaler = MinMaxScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
# Train Model
if algorithm == 'gradient_descent':
nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu',
algorithm = algorithm, max_iters = iter,
bias = True, is_classifier = True, learning_rate = 0.0001,
early_stopping = True, max_attempts = 10)#, clip_max = 5)
if algorithm == 'random_hill_climb':
nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu',
algorithm = algorithm, max_iters = iter,
bias = True, is_classifier = True,
early_stopping = True, max_attempts = 100)#, clip_max = 5)
if algorithm == 'simulated_annealing':
schedule = mlrose.GeomDecay(init_temp=100, decay=0.5, min_temp=0.1)
nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu',
algorithm = algorithm, max_iters = iter,
bias = True, is_classifier = True,
early_stopping = True, max_attempts = 100)#, schedule = schedule)#, clip_max = 5)
if algorithm == 'genetic_alg':
nn_model = mlrose.NeuralNetwork(hidden_nodes = [10,10], activation = 'relu',
algorithm = algorithm, max_iters = iter,
bias = True, is_classifier = True,
pop_size = 200, mutation_prob = 0.1,
early_stopping = True, max_attempts = 1000)#, clip_max = 5)
nn_model_cv = nn_model
nn_model.fit(X_train, y_train)
# Cross Validation
if cv == True:
pipeline = Pipeline([('transformer', scaler), ('estimator', nn_model_cv)])
cv_scores = cross_val_score(pipeline, X, y, cv=5, scoring='accuracy')
# Scores
y_train_pred = nn_model.predict(X_train)
y_train_accuracy = max(y_train_accuracy, accuracy_score(y_train, y_train_pred))
y_test_pred = nn_model.predict(X_test)
y_test_accuracy = max(y_test_accuracy, accuracy_score(y_test, y_test_pred))
print('Training accuracy: ', y_train_accuracy)
print('Test accuracy: ', y_test_accuracy)
if cv == True:
print('Cross validation accuracy: ', cv_scores.mean())
return y_train_accuracy, y_test_accuracy, cv_scores.mean()
return y_train_accuracy, y_test_accuracy, y_test_accuracy
def optimize_simulated_annealing(self):
nn = mlrose.NeuralNetwork(hidden_nodes=[10],
algorithm='simulated_annealing',
max_iters=25000,
learning_rate=0.5,
random_state=7106080,
curve=True)
hot_training_classes = to_array(self.training_classes, 10)
start = time.time()
nn.fit(self.training_features, hot_training_classes)
elapsed = time.time() - start
print('SA took %d ms and %d iterations' %
(elapsed * 1000, len(nn.fitness_curve)))
plot.clf()
plot.plot(nn.fitness_curve)
plot.ylabel('fitness')
plot.xlabel('iterations')
plot.pause(0.001)
plot.savefig('sa_fitness_curve_nn')
predicted_classes = nn.predict(self.test_features)
predicted_classes_not_array = from_array(predicted_classes)
print(
sklearn.metrics.classification_report(self.test_classes,
predicted_classes_not_array))
plot_roc_curves(self.test_classes, nn.predicted_probs)
plot.savefig('sa_roc_curve')
print(
sklearn.metrics.confusion_matrix(self.test_classes,
predicted_classes_not_array))
def run_random_hill_experiment_2(x_data, y_data, hidden_nodes):
train_scores = []
test_scores = []
train_accs, test_accs = [], []
train_times = []
restarts = [0, 1, 10, 100]
FPs, FNs = [], []
random_state = 0
for restart in restarts:
print(restart)
clf = mlrose.NeuralNetwork(
hidden_nodes = hidden_nodes, activation = 'relu', \
algorithm = 'random_hill_climb', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.2, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = None, restarts = restart, curve = True)
curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \
return_stratified_kcv_results(clf, x_data, y_data)
train_scores.append(train_score)
test_scores.append(test_score)
train_accs.append(train_acc)
test_accs.append(test_acc)
train_times.append(train_time)
FPs.append(FP)
FNs.append(FN)
print(FPs)
print(FNs)
plt.figure(2)
plt.plot(restarts, train_scores)
plt.plot(restarts, test_scores)
plt.plot(restarts, test_accs)
plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score'])
plt.title('Random Hill Climb performance vs restarts')
plt.xlabel('Restarts', fontsize=12)
plt.ylabel('Score', fontsize=12)
plt.figure(3)
plt.plot([i for i in range(0, len(curves))], curves)
plt.title('Random Hill Climb fitness at each iteration')
plt.xlabel('Iteration', fontsize=12)
plt.ylabel('Fitness', fontsize=12)
plt.show()
plt.figure(4)
plt.plot(restarts, train_times)
plt.title('Random Hill Climb train time')
plt.xlabel('Restarts', fontsize=12)
plt.ylabel('Time (s)', fontsize=12)
plt.show()
plt.figure(5)
plt.plot(restarts, FPs)
plt.plot(restarts, FNs)
plt.legend(['Type 1 Error (FP)', 'Type 2 Error (FN)'])
plt.title('Random Hill Climb TYPE 1 (FP) and TYPE 2 (FN) Errors')
plt.xlabel('Restarts', fontsize=12)
plt.ylabel('Error Rate', fontsize=12)
plt.show()
def run_SA_experiment_1(x_data, y_data, hidden_nodes):
train_scores = []
test_scores = []
train_accs, test_accs = [], []
learning_rates = [0.001, 1, 2, 5]
print('Hidden nodes :', hidden_nodes)
for learning_rate in learning_rates:
print(learning_rate)
clf = mlrose.NeuralNetwork(
hidden_nodes = hidden_nodes, activation = 'relu', \
algorithm = 'simulated_annealing', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = learning_rate, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = 30, restarts = 0)
curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \
return_stratified_kcv_results(clf, x_data, y_data)
print(train_score)
train_scores.append(train_score)
test_scores.append(test_score)
train_accs.append(train_acc)
test_accs.append(test_acc)
plt.figure(1)
plt.plot(learning_rates, train_scores)
plt.plot(learning_rates, test_scores)
plt.plot(learning_rates, test_accs)
plt.legend(['train', 'test', 'acc'])
plt.show()
def main():
# Load the Wine dataset
full_path = 'data/winequality-white.csv'
X, y = load_dataset(full_path)
#Split data into training and test sets
X_train_scaled, X_test_scaled, y_train_hot, y_test_hot = train_test_split(
X, y, test_size=0.2, random_state=RANDOM_STATE)
# Initialize neural network object and fit object
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [4], activation = 'relu', \
algorithm = 'random_hill_climb', max_iters = 3000, \
bias = True, is_classifier = True, learning_rate = 0.5, \
early_stopping = True, clip_max = 5, restarts=2, max_attempts = 100, \
random_state = RANDOM_STATE)
nn_model1.fit(X_train_scaled, y_train_hot)
# Predict labels for train set and assess accuracy
y_train_pred = nn_model1.predict(X_train_scaled)
y_train_accuracy = accuracy_score(y_train_hot, y_train_pred)
print("Train accuracy:", y_train_accuracy)
# Predict labels for test set and assess accuracy
y_test_pred = nn_model1.predict(X_test_scaled)
y_test_accuracy = accuracy_score(y_test_hot, y_test_pred)
print("Test accuracy:", y_test_accuracy)
def run_GA_experiment_1(x_data, y_data, hidden_nodes):
train_scores = []
test_scores = []
train_accs, test_accs = [], []
train_times = []
iters = [100, 500, 1000, 5000, 10000]
FPs, FNs = [], []
for iter in iters:
print(iter)
clf = mlrose.NeuralNetwork(
hidden_nodes = hidden_nodes, activation = 'relu', \
algorithm = 'genetic_alg', max_iters = iter, \
bias = True, is_classifier = True, learning_rate = 0.2, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = 30, pop_size = 100, mutation_prob = 0.1, curve = True)
curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \
return_stratified_kcv_results(clf, x_data, y_data, last_curve= True)
train_scores.append(train_score)
test_scores.append(test_score)
train_accs.append(train_acc)
test_accs.append(test_acc)
train_times.append(train_time)
FPs.append(FP)
FNs.append(FN)
print(FPs)
print(FNs)
plt.figure(1)
plt.plot(iters, train_scores)
plt.plot(iters, test_scores)
plt.plot(iters, test_accs)
plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score'])
plt.title('Genetic Algorithm performance vs max iterations')
plt.xlabel('Max iterations', fontsize=12)
plt.ylabel('Score', fontsize=12)
plt.figure(2)
plt.plot([i for i in range(0, len(curves))], curves)
plt.title('Genetic Algorithm fitness at each iteration')
plt.xlabel('Iteration', fontsize=12)
plt.ylabel('Fitness', fontsize=12)
plt.show()
plt.figure(3)
plt.plot(iters, train_times)
plt.title('Genetic Algorithm train time')
plt.xlabel('Iteration', fontsize=12)
plt.ylabel('Time (s)', fontsize=12)
plt.show()
plt.figure(4)
plt.plot(iters, FPs)
plt.plot(iters, FNs)
plt.legend(['Type 1 Error (FP)', 'Type 2 Error (FN)'])
plt.title('Genetic Algorithm TYPE 1 (FP) and TYPE 2 (FN) Error Rates')
plt.xlabel('Max iterations', fontsize=12)
plt.ylabel('Error Rate', fontsize=12)
plt.show()
def main():
# Load the Wine dataset
full_path = 'data/winequality-white.csv'
X, y = load_dataset(full_path)
#Split data into training and test sets
X_train_scaled, X_test_scaled, y_train_hot, y_test_hot = train_test_split(
X, y, test_size=0.2, random_state=RANDOM_STATE)
# Initialize neural network object and fit object
t_bef = time.time()
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [2], activation = 'tanh', \
algorithm = 'genetic_alg', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.0001, \
early_stopping = True, clip_max = 5, pop_size=900, mutation_prob=0.1, max_attempts = 100, \
random_state = RANDOM_STATE)
nn_model1.fit(X_train_scaled, y_train_hot)
t_aft = time.time()
curve = nn_model1.fitness_curve
# Predict labels for train set and assess accuracy
y_train_pred = nn_model1.predict(X_train_scaled)
y_train_accuracy = accuracy_score(y_train_hot, y_train_pred)
print("Train accuracy:", y_train_accuracy)
# Predict labels for test set and assess accuracy
y_test_pred = nn_model1.predict(X_test_scaled)
y_test_accuracy = accuracy_score(y_test_hot, y_test_pred)
print("Test accuracy:", y_test_accuracy)
def run_random_hill_experiment_1(x_data, y_data, hidden_nodes):
train_scores = []
test_scores = []
train_accs, test_accs = [], []
learning_rates = [0.001, 0.01, 0.1, 0.2, 0.5, 1]
for learning_rate in learning_rates:
print(learning_rate)
clf = mlrose.NeuralNetwork(
hidden_nodes = hidden_nodes, activation = 'relu', \
algorithm = 'random_hill_climb', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = learning_rate, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = 30, restarts = 0)
curve, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \
return_stratified_kcv_results(clf, x_data, y_data)
train_scores.append(train_score)
test_scores.append(test_score)
train_accs.append(train_acc)
test_accs.append(test_acc)
plt.figure(1)
plt.plot(learning_rates, train_scores)
plt.plot(learning_rates, test_scores)
plt.plot(learning_rates, test_accs)
plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score'])
plt.title('Random Hill Climb performance vs learning rates')
plt.xlabel('Learning rate', fontsize=12)
plt.ylabel('Score', fontsize=12)
plt.show()
def MLP_rhc(X, y, X_test, y_test):
print('Running Randomized Hill Climbing')
# Perform Random Hill Climbing
train_errors = []
test_errors = []
for i in range(0, 6):
print(i)
clf_rhc = mlrose.NeuralNetwork(hidden_nodes=[8, 8],
activation='tanh',
is_classifier=True,
algorithm='random_hill_climb',
max_iters=500,
bias=True,
learning_rate=0.01,
early_stopping=False,
clip_max=100.0,
restarts=i,
max_attempts=10,
curve=True,
random_state=1)
clf_rhc = clf_rhc.fit(X, y)
train_errors.append(clf_rhc.loss)
test_errors.append(clf_rhc.score(X_test, y_test))
clf_rhc = mlrose.NeuralNetwork(hidden_nodes=[8, 8],
activation='tanh',
is_classifier=True,
algorithm='random_hill_climb',
max_iters=500,
bias=True,
learning_rate=0.01,
early_stopping=False,
clip_max=100.0,
restarts=1,
max_attempts=10,
curve=True,
random_state=1)
clf_rhc = clf_rhc.fit(X, y)
pred_score_rhc = [clf_rhc.score(X_test, y_test)]
PlotData(list(range(len(clf_rhc.fitness_curve))), clf_rhc.fitness_curve,
list(range(len(pred_score_rhc))), pred_score_rhc,
'Error vs Iterations')
return clf_rhc, train_errors, test_errors
def nn_ga(train_x, train_y, test_x, test_y):
activations = ['identity', 'relu', 'sigmoid', 'tanh']
markers = ['o', 'x', '^', 's']
colors = ['g', 'b', 'r', 'c', 'm', 'k']
pop_sizes = [50, 200]
mutation_probs = [0.1, 0.5]
legend_items = [
mlines.Line2D([0], [0], color='g', lw=2),
mlines.Line2D([0], [0], color='b', lw=2),
mlines.Line2D([0], [0], color='r', lw=2),
mlines.Line2D([0], [0], color='c', lw=2),
mlines.Line2D([0], [0], color='m', lw=2),
mlines.Line2D([0], [0], color='k', lw=2)
]
legend_labels = [
'pop 50, prob 0.1', 'pop 200, prob 0.1', 'pop 50, prob 0.5',
'pop 200, prob 0.5'
]
counter = 0
for index, mutation_prob in enumerate(mutation_probs):
for jindex, pop_size in enumerate(pop_sizes):
nn_model = mlrose.NeuralNetwork(hidden_nodes=[8, 6],
activation='tanh',
algorithm='genetic_alg',
pop_size=pop_size,
mutation_prob=mutation_prob,
max_iters=1000,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=1000,
max_attempts=100,
random_state=3,
curve=True)
fitted_model = nn_model.fit(train_x, train_y)
plt.plot(fitted_model.fitness_curve, c=colors[counter])
counter += 1
plt.title("NN Training: GA")
plt.xlabel("No. of Iterations")
plt.ylabel("Fitness")
plt.legend(legend_items, legend_labels, loc="lower right")
plt.tight_layout()
plt.savefig("nn_ga")
plt.cla()
def run_rhc_nn(name, x_train, x_test, y_train, y_test):
print ("Working on RHC NN...")
report_name = "reports/{}_nn_rhc_output.txt".format(name)
sys.stdout = open(report_name, "w")
rhc_nn = mlrose.NeuralNetwork(hidden_nodes = [11], algorithm = 'random_hill_climb', max_iters = 1000)
run_optimized(rhc_nn, x_train, y_train, x_test, y_test)
sys.stdout = sys.__stdout__
def rhc(self, max_iter=10000, restarts=0):
learner = mlrose.NeuralNetwork(hidden_nodes=[2],
activation='relu',
algorithm='random_hill_climb',
max_iters=max_iter,
bias=True,
learning_rate=3e-03,
early_stopping=True,
max_attempts=1000,
restarts=restarts)
return learner
def run_sa_nn(name, x_train, x_test, y_train, y_test):
print ("Working on SA NN...")
report_name = "reports/{}_nn_sa_output.txt".format(name)
sys.stdout = open(report_name, "w")
sa_nn = mlrose.NeuralNetwork(hidden_nodes = [11], algorithm = 'simulated_annealing', max_iters = 1000)
run_optimized(sa_nn, x_train, y_train, x_test, y_test)
sys.stdout = sys.__stdout__
def model_complexity_exp_restart_rhc(self):
#TODO should we create a new learner object??
scoring = 'accuracy'
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \
algorithm = 'random_hill_climb', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.0001, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = 3, restarts = 0)
expHelper = ExperimentHelper(self.splitter, nn_model1, 'RHC')
param_range = [1, 10, 50, 100, 200, 300, 400, 500]
expHelper.model_complexity_exp('restarts', param_range)
def model_complexity_exp(self):
scoring = 'accuracy'
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \
algorithm = 'random_hill_climb', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.0001, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = 3)
expHelper = ExperimentHelper(self.splitter, nn_model1, 'RHC')
param_range = np.array([0.0001, 0.001, 0.002, 0.003, 0.005, 0.008])
#param_range = np.array([100, 200,300,400, 500])
expHelper.model_complexity_exp('learning_rate', param_range)
def model_complexity_exp_max_attempt_ga(self):
#TODO should we create a new learner object??
scoring = 'accuracy'
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \
algorithm = 'genetic_alg', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.0001, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
pop_size= 2000, random_state = 3)
expHelper = ExperimentHelper(self.splitter, nn_model1, 'GA')
param_range = [20, 50, 100, 200, 300]
expHelper.model_complexity_exp('max_attempts', param_range)
def run(self, n=1):
"""
n : the number of runs to do
returns best_fitnesses (list), learning_curves (list)
"""
log_loss_curves = []
train_acc = []
test_acc = []
train_f1 = []
test_f1 = []
fitted_weights = []
loss = []
for i in np.arange(n):
X_train, X_test, y_train, y_test = data_helper.train_test_split(
self.X, self.y, seed=None, test_size=self.test_size)
model = mlrose.NeuralNetwork(hidden_nodes=self.hidden_nodes,
activation=self.activation,
algorithm=self.algorithm,
max_iters=self.max_iters,
bias=True,
is_classifier=True,
learning_rate=self.learning_rate,
early_stopping=False,
clip_max=self.clip_max,
restarts=self.restarts,
schedule=self.schedule,
pop_size=self.pop_size,
mutation_prob=self.mutation_prob,
max_attempts=self.max_attempts,
curve=True)
model.fit(X_train, y_train)
y_train_pred = model.predict(X_train)
y_train_accuracy = metrics.accuracy_score(y_train, y_train_pred)
train_acc.append(y_train_accuracy)
train_f1.append(
metrics.f1_score(y_train, y_train_pred, average='weighted'))
y_test_pred = model.predict(X_test)
y_test_accuracy = metrics.accuracy_score(y_test, y_test_pred)
test_acc.append(y_test_accuracy)
test_f1.append(
metrics.f1_score(y_test, y_test_pred, average='weighted'))
log_loss_curve = np.array(model.fitness_curve) * -1
log_loss_curves.append(log_loss_curve)
# print(model.fitted_weights.shape)
# fitted_weights.append(model.fitted_weights)
loss.append(model.loss)
# print("train ", train_acc)
# print("test", test_acc)
# print(np.array(log_loss_curves).shape)
return train_acc, train_f1, test_acc, test_f1, np.array(
log_loss_curves), loss
def create_nn_schedule(decay):
if decay == 'Geom':
schedule = mlrose.GeomDecay()
if decay == 'Exp':
schedule = mlrose.ExpDecay()
if decay == 'Arith':
schedule = mlrose.ArithDecay()
return mlrose.NeuralNetwork(hidden_nodes=[10],
algorithm='simulated_annealing',
max_iters=1000,
schedule=schedule,
random_state=7106080)
def ga(self, max_iter=10000, mutation_prob=0.1, pop_size=200):
learner = mlrose.NeuralNetwork(hidden_nodes=[2],
activation='relu',
algorithm='genetic_alg',
max_iters=max_iter,
bias=True,
learning_rate=3e-03,
early_stopping=True,
max_attempts=100,
mutation_prob=mutation_prob,
pop_size=pop_size)
return learner
def model_complexity_exp_epoch_no_stop_sa(self):
#TODO should we create a new learner object??
scoring = 'accuracy'
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \
algorithm = 'simulated_annealing', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.0001, \
early_stopping = False, clip_max = 5, max_attempts = 100, \
random_state = 3)
expHelper = ExperimentHelper(self.splitter, nn_model1, 'SA')
param_range = [1, 10, 50, 100, 200, 500]
expHelper.model_complexity_exp('max_iters', param_range)
def sa(self, max_iter=10000, schedule=None):
if schedule is None:
schedule = mlrose.ExpDecay()
learner = mlrose.NeuralNetwork(hidden_nodes=[2],
activation='relu',
algorithm='simulated_annealing',
max_iters=max_iter,
bias=True,
learning_rate=3e-03,
early_stopping=True,
max_attempts=1000,
schedule=schedule)
return learner
def nn_sa_temps(train_x, train_y, test_x, test_y):
min_temps = [1, 5]
init_temps = [100, 500]
markers = ['o', 'x', '^', 's']
colors = ['g', 'b', 'r', 'c', 'y', 'm']
counter = 0
for index, min_temp in enumerate(min_temps):
for jindex, init_temp in enumerate(init_temps):
schedule = mlrose.ExpDecay(init_temp=init_temp,
exp_const=0.001,
min_temp=min_temp)
nn_model = mlrose.NeuralNetwork(hidden_nodes=[8, 6],
activation='tanh',
algorithm='simulated_annealing',
schedule=schedule,
max_iters=1000,
bias=True,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=1000,
max_attempts=100,
random_state=3,
curve=True)
fitted_model = nn_model.fit(train_x, train_y)
label = "min_temp: " + str(min_temp) + ", init_temp: " + str(
init_temp)
plt.plot(fitted_model.fitness_curve,
c=colors[counter],
label=label)
counter += 1
plt.xlabel("No. of Iterations")
plt.ylabel("Fitness")
plt.title("NN Training: SA")
plt.legend(loc="lower right")
plt.tight_layout()
plt.savefig("nn_sa_temps")
plt.cla()
def model_complexity_exp_max_attempts_sa(self):
#TODO should we create a new learner object??
scoring = 'accuracy'
decay = mlrose.decay.ExpDecay()
nn_model1 = mlrose.NeuralNetwork(hidden_nodes = [50,50,50,50], activation = 'relu', \
algorithm = 'simulated_annealing', max_iters = 1000, \
bias = True, is_classifier = True, learning_rate = 0.0001, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
schedule = decay,
random_state = 3)
expHelper = ExperimentHelper(self.splitter, nn_model1, 'SA')
param_range = [20, 50, 100, 200, 300]
expHelper.model_complexity_exp('max_attempts', param_range)
print('completed max attempt sa nn')
def genetic(iterations):
nn_model_genetic = mlrose.NeuralNetwork(hidden_nodes=[4],
activation='relu',
algorithm='genetic_alg',
max_iters=iterations,
is_classifier=True,
learning_rate=0.0001,
early_stopping=True,
clip_max=5,
max_attempts=100,
random_state=3)
nn_model_genetic.fit(X_train, y_train)
y_test_pred = nn_model_genetic.predict(X_test_scaled)
y_test_accuracy = accuracy_score(y_test, y_test_pred)
return (y_test_pred, y_test_accuracy)
def simulated_annealing(iterations):
nn_model = mlrose.NeuralNetwork(hidden_nodes=[4],
activation='relu',
algorithm='simulated_annealing',
max_iters=iterations,
is_classifier=True,
learning_rate=0.00001,
early_stopping=True,
clip_max=5,
random_state=156)
nn_model.fit(X_train, y_train)
y_test_pred = nn_model.predict(X_test)
y_test_accuracy2 = accuracy_score(y_test, y_test_pred)
return (y_test_pred, y_test_accuracy2)
def run_GA_experiment_3(x_data, y_data, hidden_nodes):
train_scores = []
test_scores = []
train_accs, test_accs = [], []
train_times = []
pop_sizes = [50, 2000]
print('Hidden nodes :', hidden_nodes)
for pop_size in pop_sizes:
clf = mlrose.NeuralNetwork(
hidden_nodes = hidden_nodes, activation = 'relu', \
algorithm = 'genetic_alg', max_iters = 200, \
bias = True, is_classifier = True, learning_rate = 0.2, \
early_stopping = True, clip_max = 5, max_attempts = 100, \
random_state = 30, pop_size = pop_size, mutation_prob = 0.1, curve = True)
curves, train_score, test_score, train_acc, test_acc, train_time, test_time, FP, FN = \
return_stratified_kcv_results(clf, x_data, y_data, last_curve=True)
train_scores.append(train_score)
test_scores.append(test_score)
train_accs.append(train_acc)
test_accs.append(test_acc)
train_times.append(train_time)
plt.figure(7)
plt.plot(pop_sizes, train_scores)
plt.plot(pop_sizes, test_scores)
plt.plot(pop_sizes, test_accs)
plt.legend(['train F1 score', 'CV F1 score ', 'CV Accuracy score'])
plt.title('Genetic Algorithm performance vs population size')
plt.xlabel('Population sizes', fontsize=12)
plt.ylabel('Score', fontsize=12)
plt.figure(8)
plt.plot([i for i in range(0, len(curves))], curves)
plt.title('Genetic Algorithm fitness at each iteration')
plt.xlabel('Iteration', fontsize=12)
plt.ylabel('Fitness', fontsize=12)
plt.show()
plt.figure(9)
plt.plot(pop_sizes, train_times)
plt.title('Genetic Algorithm train time')
plt.xlabel('population sizes', fontsize=12)
plt.ylabel('Time (s)', fontsize=12)
plt.show()
def train_and_validate(X, y, X_t, y_t):
model = mlrose.NeuralNetwork(hidden_nodes = [50, 20, 8, 2], activation = "relu", \
bias = True, is_classifier = True, early_stopping = True, \
algorithm="gradient_descent", \
max_iters=1000, max_attempts = 100, learning_rate = 0.001)
start_train = time.time()
model.fit(X, y)
print("training time: " + str(time.time() - start_train))
print("loss: " + str(model.loss))
pred = model.predict(X)
print("training data accuracy: " +
str(round(accuracy_score(y, pred) * 100, 2)) + "%")
test_pred = model.predict(X_t)
print("testing data accuracy: " +
str(round(accuracy_score(y_t, test_pred) * 100, 2)) + "%")
def learn_rate(lr):
test_accuracy = []
train_accuracy = []
learning_rate = []
for c in lr:
#schedule=mlrose.GeomDecay(init_temp=c)
nn_model1 = mlrose.NeuralNetwork(hidden_nodes=[4, 4, 4],
activation='relu',
algorithm='genetic_alg',
max_iters=1000,
mutation_prob=0.4,
bias=True,
learning_rate=0.01,
pop_size=200,
is_classifier=True,
early_stopping=True,
clip_max=5,
max_attempts=100)
nn_model1.fit(X_train, y_train)
y_train_pred = nn_model1.predict(X_train)
y_test_pred = nn_model1.predict(X_test)
y_train_accuracy = accuracy_score(y_train, y_train_pred)
y_test_accuracy = accuracy_score(y_test, y_test_pred)
test_accuracy.append(y_test_accuracy)
train_accuracy.append(y_train_accuracy)
learning_rate.append(c)
print(test_accuracy)
#print(nn_model1.fitted_weights)
train_accuracy = np.asarray(train_accuracy)
test_accuracy = np.asarray(test_accuracy)
learning_rate = np.asarray(learning_rate)
line1, = plt.plot(learning_rate,
train_accuracy,
color='r',
label='train_accuracy_uniform')
line2, = plt.plot(learning_rate,
test_accuracy,
color='b',
label='test_accuracy_uniform')
plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})
plt.ylabel('Accuracy_score')
plt.xlabel('mutation probability')
return None
def model(algorithm,
x_train,
y_train,
x_test,
iterations=100,
activation='relu'):
nn = mlrose.NeuralNetwork(
hidden_nodes=[4],
activation=activation,
algorithm=algorithm,
max_iters=iterations,
is_classifier=True,
learning_rate=0.0001,
# random_state = 2
)
nn.output_activation = mlrose.activation.sigmoid
nn.fit(x_train, y_train)
y_pred = nn.predict(x_test)
return y_pred
