def NN_fit(train_set, val_set, nn=5, epochs=10, width=10, layers=2):
from NN import NeuralNetwork
last_error = 100
last_predicated = None
fnn = None
for x in range(nn):
nn = NeuralNetwork()
nn.train(
train_set,
val_set,
epochs=epochs,
width=width,
layers=layers,
batch_size=20,
learning_rate=0.001,
)
predicted_y, _ = nn.predict(train_set.X)
logger.info(
f"NN train MSE {mean_squared_error(train_set.Y, predicted_y)}")
predicted_y, _ = nn.predict(val_set.X)
error = mean_squared_error(val_set.Y, predicted_y)
logger.info(f"NN dev MSE {error}")
if error < last_error:
last_error = error
fnn = nn
return fnn
def train(iterations, rate):
global training_inputs, temp_training_outputs, test_inputs, test_outputs
NeuralNetwork.train(NumberNet,
training_inputs=training_inputs,
training_outputs=temp_training_outputs,
training_iterations=iterations,
learning_rate=rate)
def generate():
Digit_NN = NeuralNetwork(no_of_in_nodes=image_pixels,
no_of_out_nodes=10,
no_of_hidden_nodes=100,
learning_rate=0.1)
'''
Letter_NN = NeuralNetwork(no_of_in_nodes = image_pixels,
no_of_out_nodes = 26,
no_of_hidden_nodes = 100,
learning_rate = 0.1)
'''
Meta_NN = NeuralNetwork(Digit_NN.no_of_out_nodes, 2, 100, 0.05)
#Train Digit NN
for i in range(len(digits_train_imgs)):
Digit_NN.train(digits_train_imgs[i], digits_train_labels_one_hot[i])
#Display Statistics for Digits
corrects, wrongs = Digit_NN.evaluate(digits_train_imgs,
digits_train_labels)
print("accuracy train: ", corrects / (corrects + wrongs))
corrects, wrongs = Digit_NN.evaluate(digits_test_imgs, digits_test_labels)
print("accuracy: test", corrects / (corrects + wrongs))
#Train MetaNN, save NN output vectors to be evaluated later
for i in range(len(mixed_train_imgs)):
Meta_NN.train(np.sort(Digit_NN.run(mixed_train_imgs[i]).T),
mixed_train_labels[i])
#Display Statistics for Meta
#TODO: Investigate whether this has redundant code
corrects, wrongs = Meta_NN.metaEval(Digit_NN, mixed_train_imgs,
mixed_train_labels)
train_accuracy = corrects / (corrects + wrongs)
print("Train Accuracy: ", train_accuracy)
print("Train Confusion Matrix: ")
print(
Meta_NN.meta_confusion_matrix(Digit_NN, mixed_train_imgs,
mixed_train_labels, mixed_train_values))
corrects, wrongs = Meta_NN.metaEval(Digit_NN, mixed_test_imgs,
mixed_test_labels)
test_accuracy = corrects / (corrects + wrongs)
print("Test Accuracy: ", test_accuracy)
print("Test Confusion Matrix: ")
print(
Meta_NN.meta_confusion_matrix(Digit_NN, mixed_test_imgs,
mixed_test_labels, mixed_test_values))
return train_accuracy, test_accuracy, Digit_NN, Meta_NN
def trainOR():
Network0 = NeuralNetwork(sturcture=[2, 3, 1], learningRate=0.5)
for i in range(1000):
Network0.train([0, 0], [0])
Network0.train([0, 1], [1])
Network0.train([1, 0], [1])
Network0.train([1, 1], [1])
print(i, "", round((1 - Network.costFunction) * 100, 0))
pass
print(round((1 - Network0.costFunction) * 100, 2))
Network0.answer([0, 0])
print(Network0.neurons[2][0].getOutput())
pass
def cross(Father, Mother, training_inputs, training_groundtruth, test_inputs, test_groundtruth ):
'''
This cross function is used for the genetic algorithm optimization. the inputs are two individuals, and will return a "child" with a better training performance.
This function will first generate two children. And for each child's one single feature, it will either be from the 'mother' or the 'father'
And then a quick training and testing process will be used to evalue the AUROC score.
And the 'child' with a better performance will be returned.
'''
Child_1 = []
Child_2 = []
for i in range(6):
coin = rand(-1,1)
if coin>= 0:
Child_1.append(Father[i])
Child_2.append(Mother[i])
else:
Child_1.append(Mother[i])
Child_2.append(Father[i])
Child_1 = mutate(Child_1)
Child_2 = mutate(Child_2)
# build the network, make weights, and train it
NN_1 = NeuralNetwork(input_layer=68, hidden_layer= Child_1[0], output_layer=1,
lr = Child_1[1], lr_decay= Child_1[2], iteration= Child_1[5],
batch_size= Child_1[4], mf= Child_1[3])
NN_2 = NeuralNetwork(input_layer=68, hidden_layer= Child_2[0], output_layer=1,
lr = Child_2[1], lr_decay= Child_2[2], iteration= Child_2[5],
batch_size= Child_2[4], mf= Child_2[3])
NN_1.make_weights()
NN_2.make_weights()
NN_1.train(training_inputs, training_groundtruth)
NN_2.train(training_inputs, training_groundtruth)
Score_1 = AUROC_cruve(NN_1, test_inputs, test_groundtruth, Fig=False)
Score_2 = AUROC_cruve(NN_2, test_inputs, test_groundtruth, Fig=False)
if Score_1 > Score_2:
return Child_1, Score_1
else:
return Child_2, Score_2
def trainXOR():
print("\ntrain XOR")
sturcture = [2, 3, 1]
Network0 = NeuralNetwork(sturcture=sturcture, learningRate=1)
for i in range(10000):
Network0.train([0, 0], [0])
Network0.train([0, 1], [1])
Network0.train([1, 0], [1])
Network0.train([1, 1], [0])
#print(i,"",round((1-Network0.costFunction)*100, 0))
print(i, "",
round(abs(Network0.neurons[len(sturcture) - 1][0].a - 0), 3))
print(Network0.answer([0, 0]))
print(Network0.answer([1, 0]))
print(Network0.answer([0, 1]))
print(Network0.answer([1, 1]))
pass
def do_single_run(params):
nn = NeuralNetwork(params)
final_loss = nn.train(
number_of_batches=params["Run"]["NumberOfBatchesForTraining"],
learning_rate=params["Run"]["LearningRate"])
if params["General"]["Run"]["DoEvalAOnGrid"]:
a_list = nn.eval_a_on_grid(params["Grid"]["x_min"],
params["Grid"]["x_steps"],
params["Grid"]["x_step_size"])
do_a_graph(params, a_list)
monte_carlo_price, monte_carlo_std = nn.monte_carlo_price(
number_of_batches=params["Run"]["NumberOfBatchesForEval"])
print("real price: ", nn.bachelier_call_price())
print("monte_carlo_price: ", monte_carlo_price)
print("monte_carlo_std: ", monte_carlo_std)
_, _, = nn.eval(number_of_batches=params["Run"]["NumberOfBatchesForEval"])
if params["General"]["Run"]["DoRobustGraphs"]:
params["General"]["Run"]["RatioRobustGraphs"] = params["General"][
"Run"]["RatioNumberOfBatchesForEval"] * params["General"]["Run"][
"RatioNumberOfBatchesForTraining"]
do_robust_graphs(params, nn)
nn.sess.graph.finalize()
tf.reset_default_graph()
nn.sess.close()
tf.reset_default_graph()
conf_type, diffusion_type = NeuralNetwork.get_conf_and_diffusion_types(
params["Conf"], params["Diffusion"]["Type"])
a_type = "a_" + params["a"]
file_path_final_loss = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/graph_" + conf_type + "_" + diffusion_type + "_final_loss.txt"
f = open(file_path_final_loss, "w")
file_string = "Var(Z) for single run is: " + str(final_loss)
f.write(file_string)
f.close()
'target': [0]
}]
def drawPrediction():
resolution = 10
cols = rows = int(side / resolution)
for i in range(cols):
for j in range(rows):
x1 = i / cols
x2 = j / rows
color = int(brain.predict([x1, x2])[0][0] * 255)
rect = pygame.draw.rect(
screen, (color, color, color),
(i * resolution, j * resolution, resolution, resolution))
pygame.display.flip()
while running:
event = pygame.event.poll()
if event.type == pygame.QUIT:
brain.saveState('XOR.state.json')
running = 0
if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:
brain = NeuralNetwork(2, 4, 1)
for _ in range(1000):
data = np.random.choice(training_data)
brain.train(data['input'], data['target'])
drawPrediction()
def genetic_algorithm(training_inputs, training_groundtruth, test_inputs, test_groundtruth,
num_population,times, invasion, hidden_nodes, lr, lr_decay, mf, batch_size, epoch):
'''
In this function, I will perform the genetic_algorithm to find out the best combination of hyperparameters for the training.
[training_inputs, training_groundtruth, test_inputs, test_groundtruth] is a training, testing set which are obtained for evalute the training performance.
num_population: the number of random candidates with the random number of feathers.
times: total times of "cross" for the parents to exchange the features which uis aiming at getting 'progeny" with the better performance.
invasion: number of new "invasion" individuals, besides the progeny got from the 'cross', for each time I also introduce some new individuals with different features,
which is aimming at making the whole population get more information for optimization.
[hidden_nodes, lr, lr_decay, mf, batch_size, epoch] are the six features i planned to use as the hyperparameter features that needs to be optimized
They are all put in as the two-element list, which represents the range. In the function, I will use rand funvtion to generate a random number between the range.
'''
# generate the parents population
print('generating '+ str(num_population)+' individuals')
# makeing sure the input are correct
assert hidden_nodes[0] < hidden_nodes[1], 'something went wrong!'
assert lr[0] < lr[1], 'something went wrong!'
assert lr_decay[0] < lr_decay[1], 'something went wrong!'
assert mf[0] < mf[1], 'something went wrong!'
assert batch_size[0] < batch_size[1], 'something went wrong!'
assert epoch[0] < epoch[1], 'something went wrong!'
# generating a bunch of individuals based on the range that provided
individuals_genom = []
individuals_phyno = []
for i in range(num_population):
# randonly generate the feature
_hidden_nodes = int(rand(hidden_nodes[0],hidden_nodes[1]))
_lr = rand(lr[0], lr[1])
_lr_decay = rand(lr_decay[0], lr_decay[1])
_mf = rand(mf[0],mf[1])
_batch_size = int(rand(batch_size[0],batch_size[1] ))
_epoch = int(rand(epoch[0], epoch[1]))
# build an individual and put it into the whole set
individuals_genom.append( [_hidden_nodes, _lr, _lr_decay, _mf, _batch_size, _epoch])
NN = NeuralNetwork(input_layer=68, hidden_layer= _hidden_nodes, output_layer=1,
lr = _lr, lr_decay= _lr_decay, iteration= _epoch,
batch_size= _batch_size, mf= _mf)
NN.make_weights()
NN.train(training_inputs, training_groundtruth)
# also store the individual's performance in a list vector
individuals_phyno.append(AUROC_cruve(NN, test_inputs, test_groundtruth, Fig=False))
# take the best performance people and keep it for the next generation
my_phyno = max(individuals_phyno)
idx = individuals_phyno.index(my_phyno)
my_genome = individuals_genom[idx]
n = invasion
# begin the cross, do N times of cross
for t in range(times):
print('For the time '+str(t)+' the best candidates and the best result is')
print(my_genome)
print(my_phyno)
if t >=1:
if len(individuals_genom) % 2 == 0: # add the new invasion people,
# make sure that the number is even
add = n
else:
add = n+1
for i in range(add):
_hidden_nodes = int(rand(hidden_nodes[0],hidden_nodes[1]))
_lr = rand(lr[0] , lr[1])
_lr_decay = rand(lr_decay[0], lr_decay[1])
_mf = rand(mf[0],mf[1])
_batch_size = int(rand(batch_size[0],batch_size[1] ))
_epoch = int(rand(epoch[0], epoch[1]))
individuals_genom.append( [_hidden_nodes, _lr, _lr_decay, _mf, _batch_size, _epoch])
NN = NeuralNetwork(input_layer=68, hidden_layer= _hidden_nodes, output_layer=1,
lr = _lr, lr_decay= _lr_decay, iteration= _epoch,
batch_size= _batch_size, mf= _mf)
NN.make_weights()
NN.train(training_inputs, training_groundtruth)
individuals_phyno.append(AUROC_cruve(NN, test_inputs, test_groundtruth, Fig=False))
#
next_generation_genom = []
next_generation_phyno = []
next_generation_genom.append(my_genome)
next_generation_phyno.append(my_phyno)
assert len(individuals_genom) %2 == 0, 'something wrong!'
for i in range(int(len(individuals_genom)/2)):
child_genom, child_phyno = cross(individuals_genom[i*2],individuals_genom[ i*2 +1],
training_inputs, training_groundtruth, test_inputs, test_groundtruth)
# get the child, put the child into the next generation
next_generation_genom.append(child_genom)
next_generation_phyno.append(child_phyno)
# next is now the parents
individuals_genom = next_generation_genom
individuals_phyno = next_generation_phyno
# shift it randomly
index = list(range(len(individuals_genom)))
random.shuffle(index)
individuals_phyno = [individuals_phyno[x] for x in index]
individuals_genom = [individuals_genom[x] for x in index]
# still get the best performance
my_phyno = max(individuals_phyno)
idx = individuals_phyno.index(my_phyno)
my_genome = individuals_genom[idx]
print('The whole crossing process is done')
my_phyno = max(individuals_phyno)
idx = individuals_phyno.index(my_phyno)
my_genome = individuals_genom[idx]
print('the best candidates and the best result is')
print(my_genome)
print(my_phyno)
x = preprocess(data)
train_x = x[:NUM_TRAIN]
train_y = y[:NUM_TRAIN]
test_x = x[-NUM_TEST:]
test_y = y[-NUM_TEST:]
params = {
'train_inputs': train_x,
'train_targets': train_y,
'layer_dimentions': [SIZE**2, 450, 250, 50, 10],
'learning_rate': 1e-3,
'iterations': 250
}
nn = NN(params)
nn.train()
prediction = nn.predict(test_x)
accuracy = accuracy_score(test_y, prediction)
precision = precision_score(test_y, prediction, average='micro')
recall = recall_score(test_y, prediction, average='micro')
f1 = f1_score(test_y, prediction, average='micro')
conf_matrix = confusion_matrix(test_y, prediction)
print("accuracy:", accuracy)
print("precision:", precision)
print("recall:", recall)
print("f1:", f1)
print("confusion matrix:\n", conf_matrix)
import pandas
import numpy
from NN import NeuralNetwork
numbers = pandas.read_csv('processed_train.csv')
#print(numbers.iloc[:,1:])
brain = NeuralNetwork(784, 100, 10, 0.3)
total = numbers.shape[0]
for index, row in numbers.iterrows():
target = row[1]
inputs = row[2:]
targets = numpy.zeros(10) + 0.01
targets[int(target)] = 0.99
brain.train(inputs, targets)
print(str(index) + '/' + str(total))
brain.saveState('digit-recognizer.state.json')