def main():
X, y = generateData()
start = timer()
test_x,test_y = generateData(1992)
clf = nn.multilayerperceptron(layers=[
nn.layer("tanh",3),
nn.layer("tanh",5),
nn.layer("tanh",5),
nn.layer("softmax",2)
],opt_function='momentum',drop_out=1)
clf = lr.logisticregression()
#clf = linear_model.LogisticRegression(solver="sag")
yt = pd.get_dummies(y,prefix='class').values
tyt = pd.get_dummies(test_y,prefix='class').values
clf.fit(test_x, test_y)
print(clf.coef_ )
timeusage = timer() - start
print("%f seconds"%timeusage)
y_ = clf.predict(X)
count = 0;
for i in range(len(y)):
if (y[i] == y_[i]): count += 1
print(count / len(y))
# Plot the decision boundary
plot_decision_boundary(lambda x: clf.predict(x),X,y)
plt.title("Multilayer Perceptron")
plt.show()
def train(self, x, opts): for i in range(len(self.ae)): self.ae[i] = nn.train(self.ae[i], x, x, opts) t = nn.feedforward(self.ae[i], x, x) x = t.a[1] x = x[:,1:]
def main():
X = np.array(([3, 5], [5, 1], [10, 2]), dtype=float)
y = np.array(([75], [82], [93]), dtype=float)
X = X / np.amax(X, axis=0)
y = y / 100
n = NeuralNetwork(X, y, [(3, "sigmoid")])
print n.forward(X)
print y
def run(): random.seed( 5 ) dataset = prepareDataSet() f = Sigmoid() nn = NeuralNetwork( [ dataset.sizeInputs, 5, dataset.sizeOutputs ], f, 0.3, 0.01 ) nn.train( dataset.inputs, dataset.outputs, 0.9, 1000 ) for i in range( len( dataset.testIns ) ): nn.step( dataset.testIns[ i ] ) print "actual:", getOutputName( dataset.testOuts[ i ] ), " network guess:", getOutputName( nn.activation[ nn.n - 1 ] )
def exercise1():
outputsMap = {(0,): 0, (1,): 1}
perceptron = NeuralNetwork(1, len(number0), outputsMap)
samples = (number0, number1)
expectations = ( (0,) , (1,) )
epochs = perceptron.trainNetwork(samples, expectations)
print("Treinamento demorou {} epocas".format(epochs))
print(perceptron)
print("Analisando o numero 0:")
print(perceptron.analyze(number0))
print("Analisando o numero 1:")
print(perceptron.analyze(number1))
print("Analisando o numero 2:")
print(perceptron.analyze(number2))
print("Analisando o numero 3:")
print(perceptron.analyze(number3))
print("Analisando o numero 4:")
print(perceptron.analyze(number4))
print("Analisando o numero 1 distorcido:")
print(perceptron.analyze(number1_dist))
print()
print(printWeigths(perceptron.neurons[0].weights))
printBinaryPixelMatrix(number1)
def exercise2():
outputsMap = defaultdict(lambda:None)
outputsMap[(1,0)] = 0
outputsMap[(0,1)] = 1
print(outputsMap[(1,1)])
perceptron = NeuralNetwork(2, len(number0), outputsMap)
samples = (number0, number1)
expectations = ( (1,0) , (0,1) )
epochs = perceptron.trainNetwork(samples, expectations)
print("Treinamento demorou {} epocas".format(epochs))
for x in perceptron.neurons:
print(x)
print("Analisando o numero 0:")
print(perceptron.analyze(number0))
print("Analisando o numero 1:")
print(perceptron.analyze(number1))
print("Analisando o numero 2:")
print(perceptron.analyze(number2))
print("Analisando o numero 3:")
print(perceptron.analyze(number3))
print("Analisando o numero 4:")
print(perceptron.analyze(number4))
print("Analisando o numero 5:")
print(perceptron.analyze(number5))
def test():
nn_input_size = 3 # actual input length will be +1 due to bias
fixed_point_10bit_precision = FixedPoint.FixedPointFormat(6,10)
nn = NeuralNetwork.createDemoFullyConnectNN(fixed_point_10bit_precision, nn_input_size)
##nn = NeuralNetwork.createDebugNN(fixed_point_10bit_precision, nn_input_size) #DEBUG
NN_on_CPU = NNs_on_CPU.initialize_NN_on_CPU(nn, fixed_point_10bit_precision)
input_vectors = []
input_vectors.append([3]*nn_input_size)
input_vectors.append([1]*nn_input_size)
target_vectors = []
target_vectors.append([1, 0, 0])
target_vectors.append([1, 0, 0])
learning_rate = 0.02
mini_batch_size = 10
# --- CPU ---#
NN_on_CPU.set_SGD_parameters(nn, mini_batch_size, learning_rate)
#--- ReCAM ---#
nn_weights_column = 0
nn_start_row = 0
NN_on_ReCAM = NNs_on_ReCAM.initialize_NN_on_ReCAM(nn_weights_column, nn_start_row)
NN_on_ReCAM.set_SGD_parameters(mini_batch_size, learning_rate)
NN_on_ReCAM.loadNNtoStorage(nn)
for training_iteration in range(1000):
##input_vector = input_vectors[0]
input_vector = input_vectors[training_iteration % 2]
##target_vector = target_vectors[0]
target_vector = target_vectors[training_iteration % 2]
#--- ReCAM ---#
(ReCAM_activations, ReCAM_deltas) = NN_on_ReCAM.SGD_train(nn, fixed_point_10bit_precision, nn_input_size, input_vector, target_vector) #DEBUG
##NN_on_ReCAM.SGD_train(nn, fixed_point_10bit_precision, nn_input_size, input_vector, target_output)
##ReCAM_pds = NN_on_ReCAM.get_NN_matrices(nn, NN_on_ReCAM.BP_partial_derivatives_column, nn_start_row) #DEBUG
ReCAM_weights = NN_on_ReCAM.get_NN_matrices(nn, nn_weights_column, nn_start_row)
print("Finished ReCAM Execution", training_iteration)
#--- CPU ---#
(CPU_pds, CPU_activations, CPU_deltas) = NN_on_CPU.SGD_train(nn, input_vector, target_vector) #DEBUG
###NN_on_CPU.SGD_train(nn, input_vector, target_output)
print("Finished CPU Execution", training_iteration)
# --- Verify weights match ---#
compare_activation_vectors(ReCAM_activations, CPU_activations, "activations") #DEBUG
compare_activation_vectors(ReCAM_deltas, CPU_deltas, "deltas") #DEBUG
#compare_NN_matrices(ReCAM_pds, CPU_pds, "partial derivatives") #DEBUG
compare_NN_matrices(ReCAM_weights, nn.weightsMatrices, "weights") #DEBUG
#storage.printArray()
#################################
#### Execute ####
#################################
#test()
def evaluate(args):
images, labels, length = TrainData.read(args.data_set, dataset = "testing")
nn = NeuralNetwork(in_size = 784, hidden_size = 300, out_size = 10)
nn.load(args.nn)
bar = Bar('Evaluating', max = length)
ok = 0
for i in range(length):
x = images[i]
y = labels[i]
if int(y) == int(nn.predicate(x)):
ok += 1
bar.next()
bar.finish()
print("{0:05d} / {1:05d} = {2:3.2f}%".format(ok, length, 100*ok/length))
def counting_network(hidden_units=30, learning_rate=0.15):
# this is the addends matrix
input_units = 14
# this is the output matrix
output_units = 13 + len(settings.strategies)
# fits to counting network, the inputs are the addends matrix for (1+2) , (2+3), etc and the outputs are (1+2)=3 (2+3)=4
NN = nn1.NeuralNetwork([input_units, hidden_units, output_units])
X_count = []
y_count = []
for i in range(1, 5):
X_count.append(nn1.addends_matrix(i, i + 1))
y_count.append(nn1.sum_matrix(i + 2))
X_count = np.array(X_count)
y_count = np.array(y_count)
NN.fit(X_count, y_count, learning_rate, 15000)
NN.update_y()
return NN
def DoubleMoonTest():
myann = NeuralNetwork('Dong Dong Network')
myann.model.SetInputLayer(2)
myann.model.SetOutputLayer('SoftMax')
myann.model.SetFullConnectLayer([(15,'ReLu'),(2,None)])
Na = 500
Nb = 500
(x,y) = initialize(10,6,-4,Na,Nb)
x = x.transpose()
yy = np.zeros([Na+Nb,2])
for i in range(Na+Nb):
if y[0,i]==1:
yy[i,0] = 1
else:
yy[i,1] = 1
myann.setSamples(x,yy)
myann.MiniBatch_Train(700,2000)
def train(args):
nn = NeuralNetwork(in_size = 784, hidden_size = 300, out_size = 10)
images, labels, length = TrainData.read(args.data_set, dataset = "training")
bar = Bar('Training', max = length)
for i in range(length):
x = images[i]
y = TrainData.to_formatted_array(labels[i])
## update
nn.fit(x, y)
bar.next()
bar.finish()
nn.save(args.nn)
print("saved to %s" % args.nn)
def DigitRecognitionTest():
X_train, y_train, X_val, y_val, X_test, y_test = load_and_extract_mnist_data.load_dataset()
X_train = X_train.reshape(np.size(X_train,0),28*28).transpose()
X_val = X_val.reshape(np.size(X_val,0),28*28).transpose()
X_test = X_test.reshape(np.size(X_test,0),28*28).transpose()
Y_train = []
for e in y_train:
tmp = [0]*10
tmp[e]=1
Y_train.append(tmp)
Y_train = np.array(Y_train).transpose()
Y_val = []
for e in y_val:
tmp = [0]*10
tmp[e]=1
Y_val.append(tmp)
Y_val = np.array(Y_val).transpose()
Y_test = []
for e in y_test:
tmp = [0]*10
tmp[e]=1
Y_test.append(tmp)
Y_test = np.array(Y_test).transpose()
myann = NeuralNetwork('Digital Recognition')
myann.model.SetInputLayer(28*28)
myann.model.SetOutputLayer('SoftMax')
myann.model.SetFullConnectLayer([(50,'ReLu'),(20,'ReLu'),(20,'ReLu'),(10,None)])
myann.setTrain(X_train,Y_train)
myann.setCVD(X_val,Y_val)
myann.setTest(X_test,Y_test)
myann.MiniBatch_Train(1000,2500,True)
def DoubleMoonTest():
myann = NeuralNetwork('Dong Dong Network')
myann.model.SetInputLayer(2)
myann.model.SetOutputLayer('SoftMax')
myann.model.SetFullConnectLayer([(15,'ReLu'),(2,None)])
Na = 500
Nb = 500
(x,y) = initialize(10,6,-4,Na,Nb)
#y[y==0]=-1 #use tanh to classify
yy = np.zeros([2,Na+Nb])
for i in range(Na+Nb):
if y[0,i]==1:
yy[0,i] = 1
else:
yy[1,i] = 1
myann.setSamples(x,yy)
#myann.setSamples(x,y)
myann.MiniBatch_Train(700,2000)
def exercise3():
outputsMap = defaultdict(lambda:None)
outputsMap[(1,0,0,0,0,0)] = 0
outputsMap[(0,1,0,0,0,0)] = 1
outputsMap[(0,0,1,0,0,0)] = 2
outputsMap[(0,0,0,1,0,0)] = 3
outputsMap[(0,0,0,0,1,0)] = 4
outputsMap[(0,0,0,0,0,1)] = 5
perceptron = NeuralNetwork(6, len(number0), outputsMap, randomWeights=True)
samples = (number0, number1, number2, number3, number4, number5)
expectations = ( (1,0,0,0,0,0), (0,1,0,0,0,0), (0,0,1,0,0,0), (0,0,0,1,0,0),
(0,0,0,0,1,0), (0,0,0,0,0,1) )
epochs = perceptron.trainNetwork(samples, expectations)
print("Treinamento demorou {} epocas".format(epochs))
for x in perceptron.neurons:
print(x)
print()
print("Analisando o numero 0:")
print(perceptron.analyze(number0))
print("Analisando o numero 1:")
print(perceptron.analyze(number1))
print("Analisando o numero 2:")
print(perceptron.analyze(number2))
print("Analisando o numero 3:")
print(perceptron.analyze(number3))
print("Analisando o numero 4:")
print(perceptron.analyze(number4))
print("Analisando o numero 5:")
print(perceptron.analyze(number5))
print("Analisando o numero 1 distorcido:")
print(perceptron.analyze(number1_dist))
def main(id_info, input_data, param_NN):
import NeuralNetwork
import SelectMusic
import numpy as np
### execute
#1. calc category vector which shows the distribution of weights, which shows the user's like
user_id = id_info[0]
category_vec = NeuralNetwork.main(user_id, input_data, param_NN)
#2. Select music category and music
selectedMusic = SelectMusic.main(user_id, category_vec)
# This stdout is sending to view page
print selectedMusic
return selectedMusic
def train(images, one_hot_labels):
import cifar10
def data_reshape(data, type_img=True):
if type_img:
return data.reshape(-1, 3072,1)
else:
return data.reshape(-1, 10, 1)
images_train, cls_idx_train, labels_train = cifar10.load_training_data()
images_train = data_reshape(images_train)
labels_train = data_reshape(labels_train, type_img=False)
training_data = [(x, y) for x, y in zip(images_train, cls_idx_train)]
images_test, cls_idx_test, lables_test = cifar10.load_test_data()
images_test = data_reshape(images_test)
lables_test = data_reshape(lables_test, type_img=False)
test_data = [(x, y) for x, y in zip(images_test, cls_idx_test)]
net = nn.nnetwork([3072, 120, 10])
net.train(training_data, 10, 40, 3.0, test_data=test_data)
def NN_on_ReCAM_test():
NN_Manager = initialize_NN_on_ReCAM()
nn_input_size = 3 # actual input length will be +1 due to bias
fixed_point_10bit = FixedPoint.FixedPointFormat(6,10)
nn = NeuralNetwork.createDemoFullyConnectNN(fixed_point_10bit, nn_input_size)
nn_weights_column = 0
nn_start_row = 0
NN_Manager.loadNNtoStorage(nn, nn_start_row, nn_weights_column)
input_column = 1
FP_MUL_column = 2
FP_accumulation_column = 3
input_start_row = nn_start_row
NN_Manager.loadInputToStorage(fixed_point_10bit, nn_input_size, input_column, input_start_row)
target_output = [1,2]
NN_Manager.loadTargetOutputToStorage(target_output, nn_start_row + nn.totalNumOfNetWeights, fixed_point_10bit, nn_weights_column)
FP_output = NN_Manager.feedforward(nn, nn_weights_column, nn_start_row, input_column, FP_MUL_column, FP_accumulation_column)
BP_output_column = FP_output[1]
BP_partial_derivatives_column = FP_MUL_column
activations_column = 1 if BP_output_column==3 else 3
BP_deltas_column = 4
BP_next_deltas_column = 5
NN_Manager.backPropagation(nn, nn_start_row, nn_weights_column, BP_output_column, BP_partial_derivatives_column,
activations_column, BP_deltas_column, BP_next_deltas_column)
############################################################
###### Execute
############################################################
#NN_on_ReCAM_test()
y_test_log_svm = np.zeros((140, 1))
for i in range(len(test)):
test_temp = test[i].T
for j in range(len(test_temp)):
y_test_log_svm[k][0] = i
x_test.append(test_temp[j].T)
k += 1
x_test = np.array(x_test)
x_test_vec = []
for i in range(len(x_test)):
x_temp = cv2.resize(x_test[i], (20, 17), interpolation=cv2.INTER_AREA)
x_test_vec.append(x_temp.flatten())
x_test_vec = np.array(x_test_vec)
print(x_test_vec.shape)
x_test = ((x_test - 128.0) / 128.0) - 1
test1 = NeuralNetwork.NeuralNet(340, 10, 10, actv='tanh')
test1.train(x_train_vec, y_train, x_test_vec, y_test)
lambda_set = [100, 0.01, 0.001, 10, 1, 0.1]
test1 = LogisticReg.MultiClassLog(20, 0.000005, lambda_set, 20000, 2)
param, x_test1 = test1.classification(x_train_vec, x_test_vec, y_test_log_svm)
test1.test(param, x_test1, y_test_log_svm)
print(y_train)
test2 = SVM.MultiClassSVM(1000, 1, 0.001)
param = test2.classification(x_train_vec, x_test_vec, y_test_log_svm)
test2.test(param, x_test_vec, y_test_log_svm)
def trainNetwork():
trainingData = loader.load()
nn.nNetwork(trainingData,'network.mat')
from Layers import Helpers
from Models.LeNet import build
import NeuralNetwork
import matplotlib.pyplot as plt
import os.path
batch_size = 50
mnist = Helpers.MNISTData(batch_size)
mnist.show_random_training_image()
if os.path.isfile(os.path.join('trained', 'LeNet')):
net = NeuralNetwork.load(os.path.join('trained', 'LeNet'), mnist)
else:
net = build()
net.data_layer = mnist
net.train(300)
NeuralNetwork.save(os.path.join('trained', 'LeNet'), net)
plt.figure('Loss function for training LeNet on the MNIST dataset')
plt.plot(net.loss, '-x')
plt.show()
data, labels = net.data_layer.get_test_set()
results = net.test(data)
accuracy = Helpers.calculate_accuracy(results, labels)
print('\nOn the MNIST dataset, we achieve an accuracy of: ' +
str(accuracy * 100) + '%')
def train_MNIST_and_extract_weights():
#Load MNIST data
mnist_data = MNIST(MNIST_path)
mnist_data.load_training()
mnist_data.load_testing()
nn_input_size = 784 # actual input length will be +1 due to bias
hidden_layer_size = 1000
nn = NeuralNetwork.createMNISTWeightExtractionNet(hidden_layer_size=hidden_layer_size, input_size=nn_input_size)
net_name = str(hidden_layer_size) + "HU"
NN_on_CPU = NNs_on_CPU_no_debug.initialize_NN_on_CPU(nn)
learning_rate = 0.05
mini_batch_size = 10
# --- CPU ---#
NN_on_CPU.set_SGD_parameters(nn, mini_batch_size, learning_rate)
total_training_epochs = 30
#print_net_weights_to_files(nn, net_name, 0.1)
cross_entropy_error = 0
for epoch_number in range(total_training_epochs):
##for training_iteration in range(len(mnist_data.train_images)):
for training_iteration in range(1000): #DEBUG
train_image = mnist_data.train_images[training_iteration]
train_label = mnist_data.train_labels[training_iteration]
target_output = [0] * 10
target_output[train_label] = 1
#--- CPU ---#
NN_on_CPU.SGD_train(nn, train_image, target_output)[-1]
output_layer_activations = NN_on_CPU.activations[-1]
FF_output = get_MNIST_class_from_output(output_layer_activations)
cross_entropy_error += -math.log(output_layer_activations[FF_output])
if training_iteration % 50 == 0:
print("Finished CPU Execution", training_iteration)
print("CPU FF output=",FF_output, ". Label=", train_label, ". Cross-Entropy Error=", cross_entropy_error/50)
cross_entropy_error = 0
# --- Verify weights match ---#
#NNs_unit_tests.compare_NN_matrices(ReCAM_weights, nn.weightsMatrices, "weights")
#aux_functions.write_to_output_file("Training iteration: ", training_iteration,
#". Target output:", target_output)
print("Training epoch: ", epoch_number)
number_of_correct_classifications = 0
CPU_FF_labels = []
##for testing_iteration in range(len(mnist_data.test_images)):
for testing_iteration in range(1000): #DEBUG
##test_image = mnist_data.test_images[testing_iteration]
test_image = mnist_data.train_images[testing_iteration] #DEBUG
CPU_FF_output = NN_on_CPU.feedforward(nn, test_image)
CPU_sample_label = get_MNIST_class_from_output(CPU_FF_output[-1])
##if CPU_sample_label == mnist_data.test_labels[testing_iteration]:
if CPU_sample_label == mnist_data.train_labels[testing_iteration]: #DEBUG
number_of_correct_classifications += 1
CPU_FF_labels.append(CPU_sample_label)
##fraction_of_correct_classifications = number_of_correct_classifications / len(mnist_data.test_images)
fraction_of_correct_classifications = number_of_correct_classifications / 1000
##aux_functions.write_to_output_file("epoch number:", epoch_number, ". CPU fraction of correct classifications:", fraction_of_correct_classifications)
print_net_weights_to_files(nn, net_name, fraction_of_correct_classifications)
##print("epoch number:", epoch_number, ". CPU fraction of correct classifications:", fraction_of_correct_classifications)
print("CPU number of correct classifications:", number_of_correct_classifications) #DEBUG
#----- Execute -----#
#train_MNIST_and_extract_weights()
filename, numHidden, holdout = parseInput(sys.argv)
# create list of data point objects from file
input, output = readFile(filename)
#print(input)
numpy.random.seed(4)
numInput = 2
#numHidden ^
numOutput = 1
h_percent = holdout/100
h_num = int(round(len(input)*h_percent))
ann = NeuralNetwork(numInput, numHidden, numOutput)
ann.setup()
# run backprop to train the network with the training set
print("Running back propagation", end="",flush=True)
for x in range(0,5000):
if x%100 is 0:
print('.', end="",flush=True)
#print(ann.backPropagation(input[0:h_num], output[0:h_num], .00000005))
#else:
ann.backPropagation(input[0:h_num], output[0:h_num], .00000005)
# test against rest of data
print("")
class AutoGame:
def __init__(self):
self.neuralNetwork = NeuralNetwork(shape)
self.loadNeuralNetwork()
self.runGame()
def loadNeuralNetwork(self):
generation = 132
path = "TrainingResult/"
for i in range(10):
name = glob(path + "Training Gen" + str(generation) + "_parent-" +
str(i) + "*")
globals()["neuralNetwork" + str(i)] = np.load(name[0])
print(name)
def gameOver(self, bird, arena):
runing = 1
pipeHeight = arena.upperPipe1.get_height()
if bird.birdPosition[1] > arena.groundPosition[1]:
runing = 0
dominant = str(arena.pipeDominant)
pipePosition = eval("arena.upperPositionPipe" + dominant)
topPosition = pipeHeight + pipePosition[1]
bottomPosition = topPosition + arena.deltaPipe
if bird.birdPosition[1] + bird.birdStay.get_height(
) <= topPosition and pipePosition[
0] <= bird.birdPosition[0] + bird.birdStay.get_width():
runing = 0
if bird.birdPosition[1] + bird.birdStay.get_height(
) >= bottomPosition and pipePosition[
0] <= bird.birdPosition[0] + bird.birdStay.get_width():
runing = 0
return runing, pipePosition
def runGame(self):
for i in range(10):
globals()["bird" + str(i)] = Bird()
globals()["bird" + str(i) + "Show"] = 1
globals()["bird" + str(i)].birdPosition = (BirdPosition[0],
np.random.randint(
200, 300))
vector = Vector
arena = Arena()
pipeHeight = arena.upperPipe1.get_height()
running = True
screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
surface = pygame.Surface(screen.get_size())
surface = surface.convert()
clock = pygame.time.Clock()
passing = [1] * 10
while running:
clock.tick(30)
arena.moveImage()
arena.drawBackground(surface)
arena.drawPipe(surface)
arena.drawBase(surface)
arena.drawScore(surface)
sum = 0
for i in range(10):
if passing[i] == 1:
globals()["bird" + str(i) +
"Show"], pipePosition = self.gameOver(
eval("bird" + str(i)), arena)
eval("bird" + str(i)).drawBIrd(surface)
eval("bird" + str(i)).movement()
passing[i] = globals()["bird" + str(i) + "Show"]
horizontalDistance, verticalDistance = vector.distance(
eval("bird" + str(i)).birdPosition, pipePosition,
pipeHeight)
input = np.array([horizontalDistance, verticalDistance])
# Working with Neural Network
feedForward = self.neuralNetwork.feedForward(
input,
eval("neuralNetwork" + str(i))[0],
eval("neuralNetwork" + str(i))[1])
eval("bird" + str(i)).neuralNetworkJump(feedForward)
elif passing[i] == 0:
continue
screen.blit(surface, (0, 0))
pygame.display.update()
for event in pygame.event.get():
if event.type == pygame.QUIT:
running = False
if np.sum(passing) == 0:
break
import matplotlib.pyplot as plt
import metrics
import Optimizer
import plot_results
import time
# (1) Set up data
nfeature = 2
m = 1000
case = "quadratic"
nclass = 2
noise = False
validperc = 0.1
Xtrain,Ytrain,Xvalid,Yvalid = example_classification.example(nfeature,m,case,nclass,noise,validperc)
# (2) Define model
model = NeuralNetwork.NeuralNetwork(nfeature)
model.add_layer(11,"tanh")
model.add_layer(8,"tanh")
model.add_layer(4,"tanh")
model.add_layer(1,"sigmoid")
# (3) Compile model and print summary
optimizer = Optimizer.Momentum(0.3,0.9)
model.compile("binarycrossentropy",optimizer)
model.summary()
# (4) Train model
epochs = 30
time_start = time.time()
history = model.fit(Xtrain,Ytrain,epochs,batch_size=64,verbose=True,validation_data=(Xvalid,Yvalid))
time_end = time.time()
print("Train time: {}".format(time_end - time_start))
# (5) Results
import numpy as np
import NeuralNetwork as nn
import csv
fname = '../ass2DataFiles/part1/iris.data'
def process_data(file):
feats = list()
labls = list()
with open(file, 'rt') as csvfile:
lines = csv.reader(csvfile)
for line in lines:
if line:
feats.append(line[:-1])
labls.append(line[-1])
return np.array(feats).astype(float), np.array(labls)
initfeats, initlabels = process_data(fname)
for i in range(8):
print("\n\nITR ", i + 1)
testlabels = nn.one_hot(initlabels)
features, labels = nn.clone_and_shuffle(initfeats, initlabels, 7)
labels = nn.one_hot(labels)
net = nn.NeuralNetwork([4, 5, 3], "sigmoid")
net.train(features, labels, 1000, momentum=0.8, learning_rate=0.05)
net.test(initfeats, testlabels)
t1 = time.time()
dimenssionMatrix = 4
for testDataNumber in arrTestData:
arrResult = []
middRes1 = []
middRes2 = []
print ("================TEST DATA NUMBER - " + str(testDataNumber) + " ==================")
for i in range(dimenssionMatrix):
arrTmp = []
for j in range(dimenssionMatrix):
print ("================PREPARE DATASET for[" + str(i) + "][" + str(j)+"]==============")
PrepareDataForNN.setFileName("Result\\"+str(dimenssionMatrix)+"x"+str(dimenssionMatrix)+"\ResultMatrix","Result\\"+str(dimenssionMatrix)+"x"+str(dimenssionMatrix)+"\DependMatrix")
PrepareDataForNN.createDataSet(i,j,testDataNumber)
PrepareDataForNN.createTestDataSet("Result\\4x4\\test\ResultMatrix",testDataNumber)
print ("================RUN NEURALNETWORK for[" + str(i) + "][" + str(j)+"]==============")
tempRes = NeuralNetwork.teachNeuralNetwork(PrepareDataForNN.countStates,testDataNumber)
arrTmp.append(tempRes)
arrResult.append(arrTmp)
tmpDepMatrix = []
fileDep = "Result\\"+str(dimenssionMatrix)+"x"+str(dimenssionMatrix)+"\DependMatrix" + str(testDataNumber)+".txt"
Main.read(fileDep, tmpDepMatrix)
diff1, diff2 = diffMatr(arrResult, tmpDepMatrix)
middRes1.append(diff1)
middRes2.append(diff2)
arrResultMatrix.append(arrResult)
writeResultMatrix(arrResultMatrix, "ResultMatrix.txt")
sum1 = 0
sum2 = 0
for i in range(len(middRes1)):
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
from bs4 import BeautifulSoup #网页文本解析
from urlparse import urljoin
import sqlite3 #数据库
from PorterStemmer import * #词干提取
import CRFPP #中文分词
import codecs
import sys
import NeuralNetwork #导入神经网络模块
neural_network = NeuralNetwork.SearchNet("db_network.db")
class Crawler(object):
#初始化crawler类并传入数据库名称
def __init__(self, dbname):
self.conn = sqlite3.connect(dbname)
self.stopwords = []
def __del__(self):
self.conn.close()
def dbcommit(self):
self.conn.commit()
#创建停用词列表
def make_stopwords(self, stopwords_file="stopwords.txt"):
for index, line in enumerate(open(stopwords_file, "rU")):
line = line.strip()
def __init__(self, canvas, tag, *args):
"""
Initializes the Creature
Args:
canvas: A Canvas object for the application's canvas
tag: A string for the tag used for Tkinter identification and grouping
args: A list of weights if the Creature has a parent
"""
self.tag = "%s%d" % ("creature-", tag)
self.x = int(random.random() *
Utils.board_width) if len(args) == 0 else args[1]
self.y = int(random.random() *
Utils.board_height) if len(args) == 0 else args[2]
self.r = int(random.random() * 255)
self.g = int(random.random() * 255)
self.b = int(random.random() * 255)
self.food = Utils.birth_food
self.water = Utils.birth_water
self.direction_facing = int(random.random() * 360)
self.speed = random.random()
self.action = round(random.random())
self.radius = (self.food + self.water) / 30
self.speed_coefficient = 10
self.canvas = canvas
self.tile = Utils.get_tile(self.x, self.y)
self.left_eye_rad = eye_rad(
random.random()) if len(args) == 0 else args[3]
self.right_eye_rad = eye_rad(
random.random()) if len(args) == 0 else args[4]
rgb_hex = Utils.rgb_to_hex(self.r, self.g, self.b)
center_x = self.x + self.radius
center_y = self.y + self.radius
left_eye_pos = self.get_eye_pos(self.left_eye_rad)
right_eye_pos = self.get_eye_pos(self.right_eye_rad)
self.body = self.canvas.create_oval(self.x,
self.y,
self.x + self.radius * 2,
self.y + self.radius * 2,
fill=rgb_hex,
tags=self.tag)
self.left_eye = self.canvas.create_line(center_x,
center_y,
left_eye_pos[0],
left_eye_pos[1],
tags=self.tag)
self.right_eye = self.canvas.create_line(center_x,
center_y,
right_eye_pos[0],
right_eye_pos[1],
tags=self.tag)
self.network = NeuralNetwork.NeuralNetwork(13, 8, 1, 11)
self.network.create_network()
if len(args) == 0:
self.network.create_weights()
else:
self.network.create_weights(args[0])
print("Birth: %s" % self.tag)
class NeuralNetworkCalculator(object):
def __init__(self, dimension, hiddenNum):
self.dim = dimension
self.dataSize = 0
self.inputs = []
for d in range(self.dim):
self.inputs.append([])
self.outputs = []
self.mse = 0.0
self.fitnessVal = []
self.runCnt = 2000
self.nn = NeuralNetwork([self.dim, hiddenNum, 1])
self.ga = None
def load(self, filename):
with open(filename, 'rb') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
self.dataSize += 1
for d in range(self.dim):
self.inputs[d].append(float(row[d]))
self.outputs.append(float(row[self.dim]))
self.X = np.array(self.inputs).T
self.Y = np.array(self.outputs)
def calcFitness(self, weight):
nY = []
for i in range(self.dataSize):
x = self.X[i, :]
nY.append(self.nn.calcFunc(weight, x)[0])
delta = self.Y - np.array(nY)
return np.dot(delta.T, delta) / self.dataSize
def calcByGA(self, population_num, geneRange, mutateVar):
chromoLen = self.nn.weight_num + self.nn.bias_num
self.ga = GeneticAlgorithm(population_num, geneRange, chromoLen,
self.calcFitness, mutateVar)
self.fitnessVal = []
for t in range(self.runCnt):
self.ga.next()
self.fitnessVal.append(self.ga.population[0].fitness)
print str(t) + " : " + str(self.ga.population[0].fitness)
self.betas = np.array(self.ga.population[0].genes)
nY = []
for i in range(self.dataSize):
x = self.X[i, :]
nY.append(self.nn.calcFunc(self.betas, x)[0])
delta = self.Y - np.array(nY)
self.mse = np.dot(delta.T, delta) / self.dataSize
def log(self, filename):
id = str(time.time())
with open(filename + "-" + id + ".txt", 'w') as file:
paramStr = "PARAM: "
paramStr += "I: " + str(self.nn.input_num) + " "
paramStr += "H: " + str(self.nn.hidden_num) + " "
paramStr += "O: " + str(self.nn.output_num) + " "
paramStr += "\n"
file.write(paramStr)
if self.ga != None:
gaStr = "GA "
gaStr += "Num: " + str(self.ga.population_num) + " "
gaStr += "Range: " + str(self.ga.geneRange) + " "
gaStr += "mutateVar: " + str(self.ga.mutateVar) + " "
gaStr += "mutateProb: " + str(self.ga.mutateProb) + "\n"
file.write(gaStr)
for fVal in self.fitnessVal:
file.write(str(fVal) + "\n")
midActivationFun = f.sigmoid
endActivationFun = f.sigmoid
midDeactivationFun = f.dsigmoid
endDeactivationFun = f.dsigmoid
lossFun = f.meanSquareErrorDerivative
#loading data
fileHelper = fh.FileHelper()
trainData = fileHelper.LoadClassificationData()
testData = fileHelper.LoadClassificationData()
maxCls = max(trainData, key= lambda x: x.cls).cls
nodes = [2,1,maxCls]
#initializing neural network
neuralNetwork = nn.NeuralNetwork(nodes, learningRate, useBiases, seed)
neuralNetwork.setMidActivationFun(midActivationFun)
neuralNetwork.setMidDeactivationFun(midDeactivationFun)
neuralNetwork.setEndActivationFun(endActivationFun)
neuralNetwork.setEndDeactivationFun(endDeactivationFun)
neuralNetwork.setLossFunction(lossFun)
errors = []
errSum = 0
#training
for _ in range(0, trainLoops):
for d in trainData:
predicted = neuralNetwork.train(d.inputData(), d.correctResult(nodes[-1]))
if d.cls != predicted.argmax() + 1:
errSum = errSum + 1
errors.append(errSum / (len(errors)+1))
seed,
threshold=Threshold,
Goldilock_zone=False,
plot_confuse_matrix=True)
elif NN_method == True:
print("Neural Network classification on NASA's KOI data")
print("--" * 55)
if hab_zone == True:
NN.NeuralNetwork(X_train,
X_test,
y_train,
y_test,
candidates,
GoldiLock,
seed,
threshold=Threshold,
Goldilock_zone=True,
plot_confuse_matrix=True)
else:
NN.NeuralNetwork(X_train,
X_test,
y_train,
y_test,
candidates,
GoldiLock,
seed,
threshold=Threshold,
Goldilock_zone=False,
[0,1,1],
[1,0,1],
[1,1,1]])
y4 = np.array([[0],[1],[1],[0]])
X = np.array([[0,0,0],
[0,1,0],
[1,0,0],
[1,1,0],
[0,0,1],
[0,1,1],
[1,0,1],
[1,1,1]])
y = np.array([[0],[1],[1],[0],[0],[1],[1],[0]])
NN = NeuralNetwork.NeuralNetwork(X,y)
NN4 = NeuralNetwork4Input.NeuralNetwork(X4,y4)
# NN4 = NeuralNetwork4Input.NeuralNetwork(X4,y4,True)
# Exemple de prediccio amb un entrenament insuficient
# Correr codi varis cops i veure com varia.
NN.predict(X)
print ""
NN4.predict(X4)
print ""
for i in range(500):
NN.feedforward()
NN.backprop()
NN4.feedforward()
NN4.backprop()
if display and i % 500 == 0:
print("Iteration {0:9d} Error: {1:0.6f}".format(i, error))
return i
#
# Run main
#
if __name__ == "__main__":
bitwidth = 3
# Create the network
bpn = NN.BackPropagationNetwork(
(bitwidth, 2, 1),
[None, TransferFunctions.tanh, TransferFunctions.linear])
# Create the data set
input = []
output = []
for n in range(2**bitwidth):
laInput = []
for digit in reversed(range(bitwidth)):
laInput.append((n & (2**digit)) >> digit)
input.append(np.array(laInput))
output.append(np.array([(1 if sum(input[-1]) == 1 else 0)]))
print("Input: {0} Output: {1}".format(input[-1], output[-1]))
# Train it!
Input = np.vstack(input)
def run():
#reads in the test data files.
X = readFile(sys.argv[1])
Y = readFile(sys.argv[2])
if len(sys.argv) >= 4:
weights = readWeights(sys.argv[3])
if not X.shape[0] == Y.shape[0]:
print "Need equal number of Inputs and Outputs"
return
else:
print str(X.shape[0]) + " data points given."
#Creates a trainer and network, created as a X input to a set of hidden to Y output.
print("This neural network will take in " + str(X.shape[1]) +
" inputs and will output " + str(Y.shape[1]) + " floats.")
numHiddenLayers = ""
while not numHiddenLayers.isdigit():
numHiddenLayers = raw_input("How many hidden layers do you want: ")
numHiddenLayers = int(numHiddenLayers)
layerNodes = (X.shape[1],)
print("Enter the number of nodes for each hidden layer")
for i in range(1, numHiddenLayers+1):
hiddenLayerNodes = ""
while not hiddenLayerNodes.isdigit():
hiddenLayerNodes = raw_input("Hidden Layer " + str(i) + ": ")
layerNodes = layerNodes + (int(hiddenLayerNodes),)
print layerNodes
layerNodes = layerNodes + (Y.shape[1],)
print layerNodes
NN = NeuralNetwork.NeuralNetwork(layerNodes)
if len(sys.argv) >= 4:
NN.set_params(weights)
train = NeuralNetwork.Trainer(NN)
# As the print states, runs a forward operation on the network with it's randomly generated weights.
raw_input("Now printing an initial run on the " + str(X.shape[0]) + " base inputs and their cost function.")
print(NN.forward(X))
print("Cost Function: " + NN.cost_function_type(X, Y))
print("Cost: " + str(NN.cost_function(X, Y)))
#Trains the network using the trainer and test data.
max_count = ""
while not max_count.isdigit():
max_count = raw_input("Training the network, then training on a monte carlo.\n# of Cycles: ")
max_count = int(max_count)
bestNN = NeuralNetwork.NeuralNetwork(layerNodes)
bestNN.set_params(NN.get_params())
#This is our monte carlo. Continually trains networks until one statisfies our conditions.
if max_count > 0:
count = 0
while np.isnan(bestNN.cost_function(X, Y)) or count < max_count:
train = NeuralNetwork.Trainer(NN)
train.train(X, Y)
if bestNN.cost_function(X, Y) > NN.cost_function(X, Y):
bestNN = NeuralNetwork.NeuralNetwork(layerNodes)
bestNN.set_params(NN.get_params())
print("New cost: " + str(bestNN.cost_function(X, Y)))
count += 1
NN = NeuralNetwork.NeuralNetwork(layerNodes)
print("Current cycle: " + str(count))
NN.set_params(bestNN.get_params())
#Print the results of the training and monte carlo.
raw_input("Now printing the final match results.")
print(np.around(NN.forward(X), decimals=2))
print("Cost function: " + str(NN.cost_function(X, Y)))
#Input control loop.
while 1:
ans = raw_input("input a command: forward <file>, save <file>, or exit: ")
# When a user inputs forward and a file, read in the file and run forward using it.
if ans.split(' ')[0] == 'forward' and len(ans.split(' ')) > 1:
print ans.split(' ')
input = readFile(ans.split(' ')[1])
print(np.around(NN.forward(input), decimals=2))
#Additional checker tool, allows for a forwarded file to be added to test data.
valid = raw_input("Is this the expected output? (y/n): ")
if valid == "n":
actualOutput = raw_input("What is the correct output: ")
try:
actualOutput = int(actualOutput)
with open(sys.argv[1], "a") as trainInput:
with open(ans.split(' ')[1], "r") as newInput:
trainInput.write(newInput.read())
trainInput.close()
newInput.close()
with open(sys.argv[2], "a") as trainOutput:
newData = ""
zeroes = 9 - actualOutput
while actualOutput > 0:
newData += "0 "
actualOutput = actualOutput - 1
newData += "1"
if zeroes == 0:
newData += "\n"
else:
newData += " "
while zeroes > 0:
if zeroes == 1:
newData += "0\n"
else:
newData += "0 "
zeroes = zeroes - 1
trainOutput.write(newData)
trainOutput.close()
print("Is added to training data. Will not be implemented until restart.")
except ValueError:
print("Invalid input.")
elif ans.split(' ')[0] == 'save' and len(ans.split(' ')) > 1:
print ans.split(' ')
filename = ans.split(' ')[1]
if filename == "default":
filename = "Weights-" + str(layerNodes)
with open(filename, "w") as weights:
print "Saving weights in " + filename
weights.write(str(NN.get_params()).replace("[", "").replace("]","")
.replace("\n", "").replace(" "," ").replace(" ", " "))
weights.close()
# Exit.
elif ans.split(' ')[0] == 'exit':
break
# Completely invalid input.
else:
print "#NopeNopeNope."
from matplotlib import pyplot
from NeuronLayer import *
from Neuron import *
from NeuralNetwork import *
nro_epochs = 5000
nn = NeuralNetwork([])
nn.createNetwork(7, 1, [8], 3)
msq_error = []
succeful_rate = []
for epoch in range(nro_epochs):
input_file = open("seed_f_trs.txt", 'r')
suma_error = 0
# train nn with the ~20% of the dataset
for line in input_file:
array = eval(line)
suma_error += nn.train_werror(array[0], array[1], 0.1) #usar 0.05
#nn.train(array[0], array[1], 0.3) # usar 0.05
input_file.close()
print("Despues de mi epoch nro: " + str(epoch))
msq_error.append(suma_error)
input_file = open("seed_f_tes.txt", 'r')
# test nn with the ~20% of the dataset
suma_succeful = 0
for line in input_file:
array = eval(line)
def __init__(self):
self.neuralNetwork = NeuralNetwork(shape)
self.loadNeuralNetwork()
self.runGame()
import time
import NeuralNetwork as nn
import digits_functions
# import importlib
# importlib.reload(module)
start = time.time()
topology = [50, 10]
bias = True
_lambda = 0.3
_momentum = 0.6
# sciezka = 'minist-digits-hog-50'
sciezka = 'minist-digits-hog-50-test'
# train_input_matrix, train_target_matrix, test_input_matrix, test_target_matrix = prepData.iris()
train_X, train_Y, test_X, test_Y = digits_functions.digits_hog()
# train_X, train_Y, test_X, test_Y = digits_functions.digits()
nn.learn(15, topology, train_X, train_Y, test_X, test_Y, _lambda, _momentum,
bias, 1, 0.001, sciezka, False, True)
nn.test(test_X, test_Y, topology, sciezka, False, False)
end = time.time()
print('\nExec time: ' + "%0.2f" % (end - start) + 's')
Write config in logfile
"""
with open(logfile,'a') as fl:
fl.write("########\n")
fl.write("Config\n")
fl.write("########\n")
for k,v in config.items():
fl.write('{0}: {1}\n'.format(k, v))
"""
Encoder
"""
X_dim = train_x.shape[1] # Input dimension
# Placeholders for input and latent space
X, z = inputs(X_dim, z_dim)
nn = NeuralNetwork(X_dim, h_dim, z_dim, transfer_fct = tf.nn.softplus)
if normalizing_flow:
# z_mu, z_log_var, z0, flow_params = nn.encoder(X, z, X_dim, h_dim, z_dim, nFlows)
z_mu, z_log_var, flow_params = nn.encoder_nf(X, z, X_dim, h_dim, z_dim, nFlows)
z_var= tf.exp(z_log_var) # Get variance
else:
z_mu, z_log_var= nn.enc_vanilla_vae(X)
z_var= tf.exp(z_log_var) # Get variance
# Sample the latent variables from the posterior using z_mu and z_logvar.
# Reparametrization trick is implicit in this step. Reference: Section 3 Kingma et al (2013).
z0 = nn.sample_z(z_mu, z_var)
"""
def lowerbound(dataset_name, image_index, game_type, eta, tau):
NN = NeuralNetwork(dataset_name)
NN.load_network()
print("Dataset is %s." % NN.data_set)
NN.model.summary()
dataset = DataSet(dataset_name, 'testing')
image = dataset.get_input(image_index)
(label, confidence) = NN.predict(image)
label_str = label
print(
"Working on input with index %s, whose class is '%s' and the confidence is %s."
% (image_index, label_str, confidence))
print("The second player is being %s." % game_type)
# path = "%s_pic/idx_%s_label_[%s]_with_confidence_%s.png" % (
# dataset_name, image_index, label_str, confidence)
# NN.save_input(image, path)
if game_type == 'cooperative':
tic = time.time()
cooperative = CooperativeAStar(image_index, image, NN, eta, tau)
cooperative.play_game(image)
if cooperative.ADVERSARY_FOUND is True:
elapsed = time.time() - tic
adversary = cooperative.ADVERSARY
adv_label, adv_confidence = NN.predict(adversary)
adv_label_str = adv_label
print(
"\nFound an adversary within pre-specified bounded computational resource. "
"\nThe following is its information: ")
print("difference between images: %s" %
(diffImage(image, adversary)))
l2dist = l2Distance(image, adversary)
l1dist = l1Distance(image, adversary)
l0dist = l0Distance(image, adversary)
percent = diffPercent(image, adversary)
print("L2 distance %s" % l2dist)
print("L1 distance %s" % l1dist)
print("L0 distance %s" % l0dist)
print("manipulated percentage distance %s" % percent)
print("class is changed into '%s' with confidence %s\n" %
(adv_label_str, adv_confidence))
path = "lb/coop/adv/idx_%s_modified_into_[%s]_with_confidence_%s.png" % (
image_index, adv_label_str, adv_confidence)
NN.save_input(adversary, path)
if eta[0] == 'L0':
dist = l0dist
elif eta[0] == 'L1':
dist = l1dist
elif eta[0] == 'L2':
dist = l2dist
else:
print("Unrecognised distance metric.")
path = "lb/coop/adv/idx_%s_modified_diff_%s=%s_time=%s.png" % (
image_index, eta[0], dist, elapsed)
NN.save_input(np.absolute(image - adversary), path, mul=50)
else:
print("Adversarial distance exceeds distance bound.")
elif game_type == 'competitive':
competitive = CompetitiveAlphaBeta(image, NN, eta, tau)
competitive.play_game(image)
else:
print("Unrecognised game type. Try 'cooperative' or 'competitive'.")
X_valid = Nn.NeuralNetwork.pre_process_data(X_valid)
print('2 ')
X_test = Nn.NeuralNetwork.pre_process_data(X_test)
print('Done!')
X_train1 = X_train[:num_training]
y_train1 = y_train[:num_training]
learning_rate = 1e-1
batch_size = 50
epochs = 100
reg = 1e-3
drop_out = 0.8
alpha = 0.7
lr_decay = 0.5
count = 10
# for i in xrange(count):
# print '(', i, '/', count, ')',
# learning_rate = 10**np.random.uniform(-3, -1)
# reg = 10**np.random.uniform(-3, -2)
# print 'lr:', learning_rate, ', r:', reg,
my_nn = Nn.NeuralNetwork(X_train.shape[1], 10, learning_rate, reg, drop_out,
alpha, lr_decay)
my_nn.add_layer(512)
my_nn.add_layer(128)
my_nn.train(X_train1, y_train1, X_valid, y_valid, epochs, batch_size)
my_nn.predict(X_test, y_test)
import NeuralNetwork
import numpy as np
if __name__ == "__main__":
neuralNetwork = NeuralNetwork.NeuralNetwork()
training_inputs = np.array([[0, 0, 1], [1, 1, 1], [1, 0, 1], [0, 1, 1]])
training_outputs = np.array([[0, 1, 1, 0]]).T
neuralNetwork.train(training_inputs, training_outputs, 10000)
print('Synaptic after training:')
print(neuralNetwork.synaptic_weights)
A = str(input("Input A"))
B = str(input("Input B"))
C = str(input("Input C"))
print('Situation', A, B, C)
value = neuralNetwork.think(np.array([A, B, C]))
print('Result ', value)
def backpropagation(exemplos,
thetas,
regularizacao,
network,
learning_rate,
debug=0):
J = 0
cont = 0
gradientes = []
D = []
for i in range(len(network) - 1):
D.append([])
for exemplo in exemplos:
cont += 1
if debug == 1:
print("Calculando gradientes com base no exemplo " + str(cont))
#entradas = np.array(exemplo[0])
saidas = np.array(exemplo[1])
# 1.1 Propaga x(i) e obtém as saídas f(x(i)) preditas pela rede
ativacao, saidas_preditas = NeuralNetwork.propagation(
exemplo, thetas, network)
# 1.2 calcula deltas para os neurônios da camada de saída
error = saidas_preditas - saidas
# cria array para armazenar os deltas
deltas = []
for i in range(len(network)):
deltas.append([])
deltas[-1] = error
if debug == 1:
print("delta" + str(len(network)))
print(error)
# 1.3 Para cada camada k=L-1…2, calcula os deltas para as camadas ocultas
for k in reversed(range(1, len(network) - 1)):
delta = np.transpose(thetas[k]).dot(
deltas[k + 1]) #.* ativacao[k] .* (1-ativacao[k])
delta = np.multiply(delta, ativacao[k])
delta = np.multiply(delta, (1 - ativacao[k]))
#; Remove o primeiro elemento de delta(l=k) (i.e., o delta associado ao neurônio de bias da camada k
deltas[k] = delta[1:]
if debug == 1:
print("delta" + str(k + 1))
print(deltas[k])
# 1.4 Para cada camada k=L-1…1, atualiza os gradientes dos pesos de cada camada com base no exemplo atual
for k in reversed(range(0, len(network) - 1)):
factor = []
for a in deltas[k + 1]:
line = []
for b in ativacao[k]:
line.append(a * b)
factor.append(line)
factor = np.array(factor)
#factor = deltas[k+1] * np.transpose(ativacao[k]) #não funciona por alguma razão
if debug == 1:
print("Gradientes de Theta" + str(k + 1) +
" com base no exemplo" + str(cont))
print(factor)
if len(D[k]) == 0:
D[k] = factor
else:
D[k] += factor
#
# 2. Calcula gradientes finais (regularizados) para os pesos de cada camada
for k in reversed(range(0, len(network) - 1)):
# 2.1 aplica regularização λ apenas a pesos não bias
theta_bias_zerado = np.copy(thetas[k])
for line in theta_bias_zerado:
line[0] = 0
Pk = np.multiply(regularizacao, theta_bias_zerado)
# 2.2 combina gradientes com regularização; divide pelo num de exemplos para calcular gradiente médio
D[k] = (1 / len(exemplos)) * (D[k] + Pk)
#4. atualiza pesos de cada camada com base nos gradientes
if debug == 1:
print(
"Dataset completo processado. Calculando gradientes regularizados")
novos_thetas = np.copy(thetas)
for k in range(0, len(network) - 1):
gradiente = np.multiply(learning_rate, D[k])
if debug == 1:
print("Gradientes finais para Theta" + str(k + 1) +
" (com regularizacao):")
print(gradiente)
gradientes.append(gradiente)
novos_thetas[k] = thetas[k] - gradiente
return novos_thetas, gradientes
import NeuralNetwork as neuralNet
import os.path
from os import path
# if file already exist then load datasets from external files
if (path.exists("TrainingDataset400.txt")
and path.exists("TestingDataset200.txt")):
print("\nLoading datasets from external file (totalSize = 600)...")
#load training and testing dataset from external txt file
trainingDataset = neuralNet.loadDatasetFromFile("TrainingDataset400.txt")
testingDataset = neuralNet.loadDatasetFromFile("TestingDataset200.txt")
else: # if file doesn't exist then generate the datasets and write it on a file
print("\nGenerating Training Datasets (totalSize = 600)...")
# Generate training Dataset (totalsize = 400)
datasetH200 = neuralNet.generateH(200)
datasetL200 = neuralNet.generateL(200)
trainingDataset = [datasetH200, datasetL200]
# Generate testing Dataset (totalSize = 200)
datasetH100 = neuralNet.generateH(100)
datasetL100 = neuralNet.generateL(100)
testingDataset = [datasetH100, datasetL100]
# write datasets into external txt file
neuralNet.writeDatasetIntoFile("TrainingDataset400.txt", trainingDataset)
neuralNet.writeDatasetIntoFile("TestingDataset200.txt", testingDataset)
print("\nPrinting Training Dataset...")
# print training Dataset
neuralNet.printDatasetArrays(trainingDataset)
print("\nPrinting Testing Dataset...")
# print testing Dataset
"""
import numpy as np
import sys
from NeuralNetwork import *
assert len(sys.argv) == 5, "Invalid number of arguments"
try:
trainFile = sys.argv[1]
testFile = sys.argv[2]
except ValueError:
print("Invalid input files")
raise SystemExit
try:
alpha =float( sys.argv[3])
numIterations = int(sys.argv[4])
except ValueError:
print("alpha or number of iterations entered could",)
print(" not be converted to integers")
raise SystemExit
NN_uno = NeuralNetwork(trainFile, testFile, alpha, numIterations)
NN_uno.trainNN()
NN_uno.test()
def main(batch_size, epochs, gd_variant, learning_rate, optimizer, layers,
layer_size, activation, regularization):
# shape = [784, 500, 100, 50, 30, 10]
shape = [784] + list(
range(layers * layer_size, layer_size - 1, -1 * layer_size)) + [10]
regularization = regularization == "True"
activations = {"sgm": nn.sgm, "relu": nn.relu, "tanh": nn.tanh}
reg_tag = "reg" if regularization else "no_reg"
name = "bs:" + str(batch_size) + "_epoch:" + str(epochs) + "_gd:" + str(
gd_variant) + "_lr:" + str(learning_rate) + "_opt:" + str(
optimizer) + "_num_lay:" + str(layers) + "_lay_size:" + str(
layer_size) + "_act:" + str(activation) + "_" + reg_tag
wandb.init(project="assignment1", name=name)
net = nn.NeuralNetwork(shape, activation=activations[activation])
train, test = tf.keras.datasets.fashion_mnist.load_data()
X_train, y_train = train
class_map = {}
for index, img_class in enumerate(y_train):
class_map[img_class] = index
if len(class_map) == 10:
break
for i in range(10):
plt.subplot(2, 5, 1 + i)
plt.imshow(X_train[class_map[i]], cmap=plt.get_cmap('gray'))
# plt.show()
wandb.log({'class_sample_plot': plt})
plt.clf()
y_train = y_train.reshape(60000, 1)
assert (X_train.shape,
y_train.shape) == ((60000, 28, 28),
(60000,
1)), "Train images were loaded incorrectly"
X_train = X_train.reshape(60000, 784)
X_test, y_test = test
y_test = y_test.reshape(10000, 1)
assert (X_test.shape,
y_test.shape) == ((10000, 28, 28),
(10000,
1)), "Test images were loaded incorrectly"
X_test = X_test.reshape(10000, 784)
(X_train, X_valid, y_train, y_valid) = train_test_split(X_train,
y_train,
stratify=y_train,
test_size=10000)
optimizers = {
"Adam": nn.Adam(net),
"NAdam": nn.NAdam(net),
"Momentum": nn.Momentum(net),
"RMSProp": nn.RMSProp(net),
"Adagrad": nn.Adagrad(net),
}
optimizer = optimizers[optimizer] if optimizer in optimizers.keys(
) else None
net.train(train_data=X_train,
train_labels=y_train,
test_data=X_test,
test_labels=y_test,
valid_data=X_valid,
valid_labels=y_valid,
gd_variant=gd_variant,
batch_size=batch_size,
epochs=epochs,
learning_rate=learning_rate,
regularization=regularization,
optimizer=optimizer)
def __init__(self, size): super(StackedAutoencoder, self).__init__() for i in range(1,len(size)): layers = (size[i - 1],size[i],size[i - 1]) sae.ae[i - 1] = nn.setup(layers)
def run_sl_algos(filename,
result_col,
full_param=False,
test_all=False,
debug=False,
rDTree=True,
pDTree={},
rknn=True,
pknn={},
rSVM=True,
pSVM={},
rNN=True,
pNN={},
rBTree=True,
pBTree={},
numFolds=10,
njobs=-1,
scalar=1,
make_graphs=False,
nolegend=False):
print(filename, '-', scalar)
start = time.time()
X_train, X_test, y_train, y_test = util.data_load(filename, result_col,
debug, scalar,
make_graphs)
print('data_load:',
time.strftime("%H:%M:%S", time.gmtime(time.time() - start)))
min_score = 1
max_score = 0
if rDTree:
runTime, train_score, test_score = DTree.train_DTree(
filename, X_train, X_test, y_train, y_test, full_param, debug,
numFolds, njobs, scalar, make_graphs, pDTree)
print('DTree: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, test_score)
min_score = min(min_score, test_score)
max_score = max(max_score, test_score)
if rknn:
runTime, train_score, test_score = knn.train_knn(
filename, X_train, X_test, y_train, y_test, full_param, debug,
numFolds, njobs, scalar, make_graphs, pknn)
print('knn: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, test_score)
min_score = min(min_score, test_score)
max_score = max(max_score, test_score)
if rBTree:
runTime, train_score, test_score = BTree.train_BTree(
filename, X_train, X_test, y_train, y_test, full_param, debug,
numFolds, njobs, scalar, make_graphs, pBTree)
print('BTree: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, test_score)
min_score = min(min_score, test_score)
max_score = max(max_score, test_score)
if rSVM:
L_score, R_Score, S_Score, P_Score = 0, 0, 0, 0
# runTime, train_score, L_score = SVM.train_svm(filename, X_train, X_test, y_train, y_test, 'linear',
# full_param, debug, numFolds, njobs, scalar, make_graphs)
# print('SVM-L: ', time.strftime("%H:%M:%S", time.gmtime(runTime)), train_score, L_score)
# In 14 data sets, the most common scoring was rbf > poly > linear > sigmoid
if len(pSVM) > 0:
runTime, train_score, R_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, pSVM['kernel'],
full_param, debug, numFolds, njobs, scalar, make_graphs, pSVM)
print('SVM: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, R_Score)
else:
# In 14 data sets, the most common scoring was rbf > poly > linear > sigmoid
runTime, train_score, R_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'rbf', full_param,
debug, numFolds, njobs, scalar, make_graphs)
print('SVM-R: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, R_Score)
if test_all or R_Score < 0.8:
runTime, train_score, P_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'poly',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('SVM-P: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, P_Score)
if test_all or max(R_Score, P_Score) < 0.9:
runTime, train_score, L_score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'linear',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('SVM-L: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, L_score)
if test_all or max(R_Score, P_Score, L_score) < 0.9:
runTime, train_score, S_Score = SVM.train_svm(
filename, X_train, X_test, y_train, y_test, 'sigmoid',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('SVM-S: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, S_Score)
overall_score = max(L_score, R_Score, S_Score, P_Score)
min_score = min(min_score, overall_score)
max_score = max(max_score, overall_score)
if rNN:
A_score, S_Score = 0, 0
if len(pNN) > 0:
runTime, train_score, R_Score = NN.train_NN(
filename, X_train, X_test, y_train, y_test, pNN['solver'],
full_param, debug, numFolds, njobs, scalar, make_graphs, pNN,
nolegend)
print('NN: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, R_Score)
else:
# In 10 out of 14 data sets adam was better than sgd, so testing it first
runTime, train_score, S_Score = NN.train_NN(
filename, X_train, X_test, y_train, y_test, 'adam', full_param,
debug, numFolds, njobs, scalar, make_graphs)
print('NN-A: ', time.strftime("%H:%M:%S", time.gmtime(runTime)),
train_score, S_Score)
if test_all or S_Score < 0.6:
runTime, train_score, A_score = NN.train_NN(
filename, X_train, X_test, y_train, y_test, 'sgd',
full_param, debug, numFolds, njobs, scalar, make_graphs)
print('NN-S: ', time.strftime("%H:%M:%S",
time.gmtime(runTime)),
train_score, A_score)
overall_score = max(A_score, S_Score)
min_score = min(min_score, overall_score)
max_score = max(max_score, overall_score)
print('Overall Variance: ', round(max_score - min_score, 4), '\n')
# seed=20190501
# )
# nn = NeuralNetwork(
# layers=[Dense(neurons=13,
# activation=Sigmoid()),
# Dense(neurons=1,
# activation=Linear())],
# loss=MeanSquaredError(),
# seed=20190501
# )
dl = NeuralNetwork(layers=[
Dense(neurons=13, activation=SigMoid()),
Dense(neurons=13, activation=SigMoid()),
Dense(neurons=1, activation=Linear())
],
loss=MeanSquaredError(),
seed=20190501)
boston = load_boston()
data = boston.data
target = boston.target
features = boston.feature_names
s = StandardScaler()
data = s.fit_transform(data)
X_train, X_test, y_train, y_test = train_test_split(data,
target,
test_size=0.3,
random_state=80718)
from Matrix import *
from Picture import *
from NeuralNetwork import *
import random
'''picture = Picture("Letters/A/A07.png")
pix = Matrix.from_matrix(picture.matrix)
pix.print()'''
nn = NeuralNetwork(225,10,2)
training_data = [
[Picture("Letters/A/A01.png").matrix, [[1],[0]]],
[Picture("Letters/A/A02.png").matrix, [[1],[0]]],
[Picture("Letters/A/A03.png").matrix, [[1],[0]]],
#[Picture("Letters/A/A04.png").matrix, [[1],[0]]],
#[Picture("Letters/A/A05.png").matrix, [[1],[0]]],
#[Picture("Letters/B/B01.png").matrix, [[0],[1]]],
#[Picture("Letters/B/B02.png").matrix, [[0],[1]]],
[Picture("Letters/B/B03.png").matrix, [[0],[1]]],
[Picture("Letters/B/B04.png").matrix, [[0],[1]]],
[Picture("Letters/B/B05.png").matrix, [[0],[1]]]]
nn.guess(Matrix.from_matrix(Picture("Letters/A/A06.png").matrix))
nn.guess(Matrix.from_matrix(Picture("Letters/B/B06.png").matrix))
for x in range(10000):
print(x)
random.shuffle(training_data)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# AutoEncoder.py
import NeuralNetwork as nn
from Utils import MatLoad
import numpy as np
if __name__ == "__main__":
X = MatLoad("XAutoencoder.np"); # Vectores por columnas (N=|cols|, M=|rows|) #
XROWS,XCOLS,M1 = X.shape[0],X.shape[1],10 # Se cuenta la unidad BIAS en la capa oculta (la entrada se asume en notación homogénea) #
units_by_layer = [XROWS,M1,XROWS-1]
theta = nn.generate_theta(units_by_layer)
Y = X[1:]
res = nn.fit(theta,X,Y,units_by_layer,nn.lineal,(X,Y,units_by_layer,nn.lineal),f_solver=nn.f,method="SLSQP",max_iter=1000000)
print "\n Detalles de convergencia \n"
print res
theta = res.x
theta = nn.get_weights_submatrices(theta,units_by_layer)
print "Vector original: ",np.matrix([[1], [0], [1], [1], [0], [1], [0], [0], [0], [0], [1], [1], [0], [1], [1], [1],[1]])
encoded = nn.forward_propagation_transform(theta[0],np.matrix([[1], [0], [1], [1], [0], [1], [0], [0], [0], [0], [1], [1], [0], [1], [1], [1],[1]]),units_by_layer,nn.lineal)
encoded = np.insert(encoded,0,1,axis=0)
print "Vector cifrado: ",encoded.tolist()
decoded = nn.forward_propagation_transform(theta[1],encoded,units_by_layer,nn.lineal)
print "Vector descifrado: ",decoded
from NeuralNetwork import *
import numpy as np
import matplotlib.pyplot as plt
# =========== 학습 ============= #
# 입력, 은닉, 출력 노드 수
input_nodes = 784
hidden_nodes = 100
output_nodes = 10
# 학습률
learning_rate = 0.1
# 신경망 인스턴스 생성
n = NeuralNetwork(input_nodes, hidden_nodes, output_nodes, learning_rate)
# mnist 학습 데이터인 csv 파일 로딩
# with open(r"./data/mnist_train.csv", "r") as training_data_file:
# training_data_list = training_data_file.readlines()
training_data_file = open(r"./data/mnist_train.csv", "r")
training_data_list = training_data_file.readlines()
training_data_file.close()
epochs = 3
# 학습시작
for e in range(epochs):
for record in training_data_list:
all_values = record.split(",")
inputs = (np.asfarray(all_values[1:]) / 255.0 * 0.99) + 0.01
def train_MNIST():
#Load MNIST data
#aux_functions.open_output_file(MNIST_path + '\\Outputs\\')
mnist_data = MNIST(MNIST_path)
mnist_data.load_training()
mnist_data.load_testing()
learning_rate = 0.02
mini_batch_size = 50
nn_input_size = 784 # actual input length will be +1 due to bias
fixed_point_10bit_precision = FixedPoint.FixedPointFormat(6, 10)
##nn = NeuralNetwork.createMNISTFullyConnectNN(fixed_point_10bit_precision, nn_input_size)
##nn = NeuralNetwork.createDemoFullyConnectNN(fixed_point_10bit_precision, nn_input_size)
nn = NeuralNetwork.createMNISTConvNet(fixed_point_10bit_precision, nn_input_size)
# --- CPU ---#
if is_CPU_active:
NN_on_CPU = NNs_on_CPU.initialize_NN_on_CPU(nn, is_activations_debug_active, is_pds_debug_active, is_deltas_debug_active, fixed_point_10bit_precision)
NN_on_CPU.set_SGD_parameters(nn, mini_batch_size, learning_rate)
# --- ReCAM ---#
if is_ReCAM_active:
nn_weights_column = 0
nn_start_row = 0
ReCAM_size = 419430400
NN_on_ReCAM = NNs_on_ReCAM.initialize_NN_on_ReCAM(nn_weights_column, nn_start_row, ReCAM_size, is_activations_debug_active, is_pds_debug_active, is_deltas_debug_active)
NN_on_ReCAM.set_SGD_parameters(mini_batch_size, learning_rate)
NN_on_ReCAM.loadNNtoStorage(nn)
total_training_epochs = 100
epoch_number = 0
training_iteration = 0
#for epoch_number in range(total_training_epochs):
# for training_iteration in range(len(mnist_data.train_images)):
train_image = fixed_point_10bit_precision.convert_array_to_fixed_point(mnist_data.train_images[training_iteration])
train_label = mnist_data.train_labels[training_iteration]
target_output = [0] * 10
target_output[train_label] = 1
#--- ReCAM ---#
if is_ReCAM_active:
if is_activations_debug_active or is_deltas_debug_active:
(ReCAM_activations, ReCAM_deltas) = NN_on_ReCAM.SGD_train(nn, fixed_point_10bit_precision, nn_input_size, train_image, target_output)
else:
NN_on_ReCAM.SGD_train(nn, fixed_point_10bit_precision, nn_input_size, train_image, target_output)
if is_CPU_active and is_ReCAM_active:
ReCAM_weights = NN_on_ReCAM.get_NN_matrices(nn, nn_weights_column, nn_start_row)
if is_pds_debug_active:
ReCAM_pds = NN_on_ReCAM.get_NN_matrices(nn, NN_on_ReCAM.BP_partial_derivatives_column, nn_start_row)
if is_ReCAM_active:
NN_on_ReCAM.storage.printHistogramsToExcel(nn, len(mnist_data.train_images))
print("Finished ReCAM Execution", training_iteration)
#--- CPU ---#
if is_CPU_active:
if is_activations_debug_active or is_pds_debug_active or is_deltas_debug_active:
(CPU_activations, CPU_pds, CPU_deltas) = NN_on_CPU.SGD_train(nn, train_image, target_output)
else:
NN_on_CPU.SGD_train(nn, train_image, target_output)
print("Finished CPU Execution", training_iteration)
# --- Verify weights match ---#
if is_ReCAM_active:
NNs_unit_tests.compare_NN_matrices(ReCAM_weights, nn.weightsMatrices, "weights")
if is_activations_debug_active:
NNs_unit_tests.compare_activation_vectors(ReCAM_activations, CPU_activations, "activations")
if is_deltas_debug_active:
NNs_unit_tests.compare_activation_vectors(ReCAM_deltas, CPU_deltas, "deltas")
if is_pds_debug_active:
NNs_unit_tests.compare_NN_matrices(ReCAM_pds, CPU_pds, "partial derivatives")
L = len(layers_dims) - 1
for l in range(1, L + 1):
parameters['W' + str(l)] = np.random.randn(layers_dims[l], layers_dims[l - 1]) * np.sqrt(
2. / layers_dims[l - 1])
parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))
return parameters
weights = initialize_parameters_he([784, 500, 11])
weights['b2'] = weights['b2'].reshape(11, 1)
weights['b1'] = weights['b1'].reshape(500, 1)
print('loading data...')
inputs = nn.LoadData(60000, 0)
random.shuffle(inputs)
for i in range(0, 60000+4078):
inputs[i].input = np.asarray(inputs[i].input)
inputs[i].input = inputs[i].input.reshape(784, 1)
print('')
print('')
print('training...')
print('')
for z in range(0, epochs):
for i in range(0, 60000+4078):
hidden1 = nn.create_hidden(weights['W1'], inputs[i].input, weights['b1'], activation='sigmoid',
dropout=True)
#!/usr/bin/env python
from flask import Flask, render_template, request, jsonify
from NeuralNetwork import *
import numpy
import pprint
pp = pprint.PrettyPrinter(indent=4)
app = Flask(__name__)
nn = NeuralNetwork(in_size = 784, hidden_size = 300, out_size = 10)
nn.load("mat.npz");
@app.route("/")
def index():
return render_template("index.html")
@app.route("/estimate", methods = ["POST"])
def estimate():
try:
x = numpy.array(request.json["input"]) / 255.0
y = int(nn.predicate(x))
return jsonify({"estimated":y})
except Exception as e:
print(e)
return jsonify({"error":e})
if __name__ == '__main__':
app.run(host='0.0.0.0')
historyWeightsHO = np.load("NN/HO.npy") # hidden two to output
historyBiasHO = np.load("NN/BHO.npy") # bias to output
# Save fitness and diversity of individuals
maxFitness = []
avgFitness = []
maxDiversity = []
avgDiversity = []
n_epochs = 500
IH = historyWeightsIH[-1][0]
IHH1 = historyBiasIHH1[-1][0] # biased to hidden one
HO = historyWeightsHO[-1][0] # hidden two to output
BiasHO = historyBiasHO[-1][0] # bias to output
nn = NeuralNetwork(16, [4], 2, tanh)
nn.weightsIH = IH
nn.biasIHH[0] = IHH1
nn.weightsHO = HO
nn.biasHO = BiasHO
# print(fitness.shape)
individual = Individual()
individual.nn = nn
# Compute the max fitness, average fitness, max diversity and average diversity through out all generations
def draw_fitness(fitness_history, display=False):
for t in range(n_epochs):
t_th_population = fitness_history[t]
end_pos = (1300, 150)
start = obj.Circle(start_pos, 10, (255, 0, 0))
end = obj.Circle(end_pos, 10, (0, 255, 0))
generation = 0
boundaries = obj.get_boundaries(w, h)
particles = obj.get_particles(start_pos, number_of_particles, number_of_rays, ray_length, ray_color, particle_size)
clock = py.time.Clock()
game = True
while game:
screen.fill((0, 0, 0))
for event in py.event.get():
if event.type == py.QUIT:
game = False
py.quit()
quit()
if event.type == py.MOUSEBUTTONDOWN:
x, y = py.mouse.get_pos()
print(x, y)
generation += 1
print('Generation {}'.format(generation))
run(screen)
particles = NN.next_generation(particles, number_of_particles, mutation_rate, number_of_rays, ray_length, ray_color, particle_size, end_pos, *start_pos)
py.display.update()
clock.tick(fps)
"""
Created on Tue Apr 26 19:01:44 2016
@author: gbayomi
"""
from NeuralNetwork import *
from Loader import *
import matplotlib.pyplot as plt
#Format the training images to [0, 1) size
test_images, test_labels = getTestingSample(10000)
test_images = test_images/255.0
#Format the testing images to [0, 1) size
testX = test_images
testY = test_labels
#Load the last saved results
W1 = np.load('files/9271W1.npy')
W2 = np.load('files/9271W2.npy')
J = np.load('files/9271J.npy')
#Run a Neural Network: input size -> 784; hidden layer->50; output->10; activation-> sigmoid
NN = NeuralNetwork(784, 50, 10, "sigmoid")
NN.loadWeights(W1, W2)
label = NN.getAccuracy(testX, testY)
plt.plot(J)
plt.xlabel(label)
plt.show()
