def main():
# Carregando os dados
full_td, _, _ = mnist_loader.load_data_wrapper()
td = full_td[:1000]
epochs = 500
print("\nDuas Camadas Ocultas:")
net = network2.Network([784, 30, 30, 10])
initial_norms(td, net)
abbreviated_gradient = [
ag[:6] for ag in get_average_gradient(net, td)[:-1]
]
f = open("initial_gradient.json", "w")
json.dump(abbreviated_gradient, f)
f.close()
shutil.copy("initial_gradient.json", "../../js/initial_gradient.json")
training(td, net, epochs, "norms_during_training_2_layers.json")
plot_training(epochs, "norms_during_training_2_layers.json", 2)
print("\nTrês Camadas Ocultas:")
net = network2.Network([784, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_3_layers.json")
plot_training(epochs, "norms_during_training_3_layers.json", 3)
print("\nQuatro Camadas Ocultas:")
net = network2.Network([784, 30, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_4_layers.json")
plot_training(epochs, "norms_during_training_4_layers.json", 4)
def run_network():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
print "Training the network using Quadratic cost function"
net = network2.Network([784, 30, 10], cost=network2.QuadraticCost)
vc_1, va_1, tc_1, ta_1 \
= net.SGD(training_data, 30, 10, 0.025, lmbda=5.0,
evaluation_data=validation_data,
monitor_evaluation_accuracy=True)
print "Training the network using Cross-Entropy cost function"
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
vc_2, va_2, tc_2, ta_2 \
= net.SGD(training_data, 30, 10, 0.025, lmbda=5.0,
evaluation_data=validation_data,
monitor_evaluation_accuracy=True)
f = open("./evaluation_other_costfunction.json", "w")
json.dump(
{
"Quadratic_Cost": [vc_1, va_1, tc_1, ta_1],
"Cross-Entropy_Cost": [vc_2, va_2, tc_2, ta_2]
}, f)
f.close()
def main():
global exit_clicked
global keyPressed
keyPressed = False
exit_clicked = False
plt.ioff()
plt.show()
global mode
mode = 3
if len(sys.argv) > 1:
mode = int(sys.argv[1])
fig = plt.plot()
plt.switch_backend('TKAgg')
figManager = plt.get_current_fig_manager()
figManager.window.state('zoomed')
if mode == 0:
print "Net 2, 10, 10, 2"
img = 0.5*np.ones((140,140))
net = network2.Network([2, 10, 10, 2], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
myobj = NNIMG(fig, img, net)
plt.gca().invert_yaxis()
plt.draw()
if mode == -1:
print "Net 2, 2, 2"
img = 0.5*np.ones((140,140))
net = network2.Network([2, 2, 2], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
myobj = NNIMG(fig, img, net)
plt.gca().invert_yaxis()
plt.draw()
if mode == 1:
train ('C:/Users/ThomasReichert/source/repos/TheLIFO/NeuralNetMINT/trainednet2')
if mode == 2:
img = np.zeros((28,28))
net = network2.load('C:/Users/ThomasReichert/source/repos/TheLIFO/NeuralNetMINT/trainednet2')
myobj = drawableIMG(fig, img, net)
if mode == 3:
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
img = np.zeros((28,28))
net = network2.load('C:/Users/ThomasReichert/source/repos/TheLIFO/NeuralNetMINT/trainednet2')
myObj = testIMGs(fig, img, net, test_data)
plt.ion()
while not(exit_clicked):
plt.pause(0.001)
print "exit"
def main():
# Load the data
full_td,_,_ = mnist_loader.load_data_wrapper()
td = full_td[:1000] # Just use the first 1000 items of training data
epochs = 500 # Number of epochs to train for
print("\nTwo hidden layers:")
net = network2.Network([784, 30, 30, 10])
initial_norms(td,net)
abbreviated_gradient = [ag[:6] for ag in get_average_gradient(net, td)[:-1]]
print ("Saving the averaged gradient for the top six neurons in each "+\
"layer.\nWARNING: This will affect the look of the book, so be "+\
"sure to check the\nrelevant material (early chapter 5).")
f = open("initial_gradient.json","w")
json.dump(abbreviated_gradient, f)
f.close()
shutil.copy("initial_gradient.json", "../../js/initial_gradient.json")
training(td,net,epochs, "norms_during_training_2_layers.json")
plot_training(epochs, "norms_during_training_2_layers.json",2)
print("\nThree hidden layers:")
net = network2.Network([784,30,30, 30, 10])
initial_norms(td, net)
training(td, net, epochs,"norms_during_training_3_layers.json")
plot_training(epochs,"norms_during_training_3_layers.json",3)
print("\nFour hidden layers:")
net = network2.Network([784, 30, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_4_layers.json")
plot_training(epochs, "norms_during_training_4_layers.json",4)
def main():
# Load the data
full_td, _, _ = mnist_loader.load_data_wrapper()
td = full_td[:1000] # Just use the first 1000 items of training data
epochs = 500 # Number of epochs to train for
####
print("\nTwo hidden layers:")
net = network2.Network([784, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_2_layers.json")
plot_training(epochs, "norms_during_training_2_layers.json", 2)
####
print("\nThree hidden layers:")
net = network2.Network([784, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_3_layers.json")
plot_training(epochs, "norms_during_training_3_layers.json", 3)
####
print("\nFour hidden layers:")
net = network2.Network([784, 30, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_4_layers.json")
plot_training(epochs, "norms_during_training_4_layers.json", 4)
def main():
full_training_data, _, _ = mnist_loader.load_data_wrapper()
training_data = full_training_data[:1000]
epoches = 500
print("\nTwo hidden layers:")
net = network2.Network([784, 30, 30, 10])
initial_norms(training_data, net)
abbreviated_gradient = [ag[:6] for ag in get_average_gradient(net, training_data)[:-1]]
print("Saving the averaged gradient for the top six nerons in each layer.")
with open("./js/initial_gradient.json", "w") as f:
json.dump(abbreviated_gradient, f)
training(training_data, net, epoches, "./js/norms_during_training_2_layers.json")
plot_training(epoches, "./js/norms_during_training_2_layers.json", 2)
print("\nThree hidden layers:")
net = network2.Network([784, 30, 30, 30, 10])
initial_norms(training_data, net)
training(training_data, net, epoches, "./js/norms_during_training_3_layers.json")
plot_training(epoches, "./js/norms_during_training_3_layers.json", 3)
print("\nFour hidden layers:")
net = network2.Network([784, 30, 30, 30, 30, 10])
initial_norms(training_data, net)
training(training_data, net, epoches, "./js/norms_during_training_4_layers.json")
plot_training(epoches, "./js/norms_during_training_4_layers.json", 4)
def init_net():
global net
if network.__name__ == 'network':
net = network.Network([784, 30, 10])
elif network.__name__ == 'network2':
net = network.Network([784, 30, 10], cost=network.CrossEntropyCost)
net.large_weight_initializer()
else:
return False
def run_network(filename, n, eta):
"""Train the network using both the default and the large starting
weights. Store the results in the file with name ``filename``,
where they can later be used by ``make_plots``.
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784, n, 10], cost=network2.CrossEntropyCost)
print ("Train the network using the default starting weights.")
default_vc, default_va, default_tc, default_ta \
= net.SGD(training_data, 30, 10, eta, lmbda=5.0,
evaluation_data=validation_data,
monitor_evaluation_accuracy=True)
print ("Train the network using the large starting weights.")
net.large_weight_initializer()
large_vc, large_va, large_tc, large_ta \
= net.SGD(training_data, 30, 10, eta, lmbda=5.0,
evaluation_data=validation_data,
monitor_evaluation_accuracy=True)
f = open(filename, "w")
json.dump({"default_weight_initialization":
[default_vc, default_va, default_tc, default_ta],
"large_weight_initialization":
[large_vc, large_va, large_tc, large_ta]},
f)
f.close()
def digit_recognize_test():
#载入数据
train = pd.read_csv('../../input/train.csv')
print('train: ' + str(train.shape))
test = pd.read_csv('../../input/test.csv')
print('test: ' + str(test.shape))
train.head()
#train: (2000, 785)
#test: (28000, 784)
# feature matrix
X = train.ix[:,1:]
# response vector
X = X.values
Y = train['label']
Y = Y.values
#训练数据
training_inputs = [np.reshape(x, (784, 1)) for x in X]
training_results = [vectorized_result(y) for y in Y]
training_data = list(zip(training_inputs, training_results))
print(np.shape(training_results))
#测试数据
test_data = list(test.values)
predit_inputs = [np.reshape(x, (784, 1)) for x in test_data]
net = network2.Network([784,50,10])
predictions = net.SGD_kaggle(training_data, 60,10,0.1,lmbda = 5.0,test_data = predit_inputs)
#输出数据
result = pd.DataFrame({'ImageId': list(range(1,len(predictions)+1)), 'Label': predictions})
result.to_csv('../../output/dr_result.csv', index=False, header=True)
print("finished!")
def get_string_from_nn(all_letters):
net = network2.Network([1024, 30, 66], cost=network2.CrossEntropyCost)
biases_saved = np.load('./training_nn/biases_weights/biases.npy',
encoding='latin1')
weights_saved = np.load('./training_nn/biases_weights/weights.npy',
encoding='latin1')
#all_letters = np.load('all_letters.npy')
#all_letters = all_letters.tolist()
word_string = ""
#pred=loadModel()
i = 0
for x in all_letters:
output = np.argmax(
net.feedforward(x,
biases_saved=biases_saved,
weights_saved=weights_saved))
#output = np.argmax(getResult(all_letters,pred))
#second stage classification below
if (output in (18, 19, 21, 29, 44, 47, 1)):
output = get_let_from_2nd_nn_ijltIL1(x)
elif (output in (12, 14, 42)):
output = get_let_from_2nd_nn_ceg(x)
word_string = word_string + get_letter(output)
i = i + 1
return word_string
def __init__(self, sentence_each):
with open("data/test.csv", "r") as f:
with open("submission.csv", "w") as g:
print >> g, "test_id,is_duplicate"
net = network2.Network([1200, 100, 2])
sess = net.restore("my-model-10")
reader = csv.reader(f, delimiter=',', quotechar='"')
next(reader)
i = 0
for line in reader:
if i % 1000 == 0:
print "Read first ", i, " test questions"
pair_id = int(line[0])
q1 = line[1]
q1 = q_to_words(q1)
#''.join(ch.lower() for ch in q1.replace(","," ") if ch.isalnum() or ch==' ').split()
q2 = line[2]
q2 = q_to_words(q2)
#''.join(ch.lower() for ch in q2.replace(","," ") if ch.isalnum() or ch==' ').split()
is_duplicate = None
vec = sentence_each.process(q1, q2, is_duplicate)
print >> g, str(i) + "," + str(
net.get_results(Dataset(vec.reshape(1200, 1).T, None),
sess)[:, 1][0])
i += 1
sess.close()
def run_network(filename, num_epochs, training_set_size=1000, lmbda=0.0):
"""Train the network for ``num_epochs`` on ``training_set_size``
images, and store the results in ``filename``. Those results can
later be used by ``make_plots``. Note that the results are stored
to disk in large part because it's convenient not to have to
``run_network`` each time we want to make a plot (it's slow).
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost())
net.large_weight_initializer()
test_cost, test_accuracy, training_cost, training_accuracy \
= net.SGD(training_data[:training_set_size], num_epochs, 10, 0.5,
evaluation_data=test_data, lmbda = lmbda,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
f = open(filename, "w")
json.dump([test_cost, test_accuracy, training_cost, training_accuracy], f)
f.close()
def train(savename):
# here is code for training the network
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784,100, 10], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
net.SGD(training_data, 30, 10, 0.5, evaluation_data=test_data, monitor_evaluation_accuracy=True)
net.save(savename)
def run_networks():
"""Train networks using three different values for the learning rate,
and store the cost curves in the file ``multiple_eta.json``, where
they can later be used by ``make_plot``.
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = test1.load_data_wrapper()
results = []
for eta in LEARNING_RATES:
print "\nTrain a network using eta = " + str(eta)
net = network2.Network([784, 30, 10])
results.append(
net.SGD(training_data,
NUM_EPOCHS,
10,
eta,
lmbda=5.0,
evaluation_data=validation_data,
monitor_training_cost=True))
f = open("multiple_eta.json", "w")
json.dump(results, f)
f.close()
def run_network(training_size, num_epochs, regularization_type, file_name):
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network2.Network([784, 30, 10],
cost=network2.CrossEntropyCost(),
regularization=regularization_type)
print("\n\nTraining network with data set size %s" % (training_size))
net.large_weight_initializer()
test_cost, test_accuracy, training_cost, training_accuracy \
= net.SGD(training_data[:training_size], num_epochs,
10, 0.5, lmbda=training_size*0.0001,
evaluation_data=test_data,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
accuracy = net.accuracy(validation_data) / 100.0
print("Accuracy on validation data was %s percent" % accuracy)
f = open(file_name, "w")
json.dump([test_cost, test_accuracy, training_cost, training_accuracy], f)
f.close()
def get_string_from_nn(all_letters):
net = network2.Network([1024, 30, 66], cost=network2.CrossEntropyCost)
biases_saved = np.load('biases.npy')
weights_saved = np.load('weights.npy')
#all_letters = np.load('all_letters.npy')
#all_letters = all_letters.tolist()
word_string = ""
i = 0
for x in all_letters:
output = np.argmax(
net.feedforward(x,
biases_saved=biases_saved,
weights_saved=weights_saved))
#second stage classification below
if (output in (18, 19, 21, 29, 44, 47, 1)):
output = get_let_from_2nd_nn_ijltIL1(x)
elif (output in (12, 14, 42)):
output = get_let_from_2nd_nn_ceg(x)
word_string = word_string + get_letter(output)
i = i + 1
return word_string
#print np.argmax(net.feedforward(test_data[502][0], biases_saved = biases_saved, weights_saved = weights_saved))
def run_network(filename, num_epoches, training_set_size=1000, lmbda=0.0):
"""
Train the network for 'num_epoches' on 'training_set_size' images, and store the results in 'filename'.
Those results can later be used by 'make_plots'.
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost())
net.large_weight_initializer()
test_cost, test_accuracy, training_cost, training_accuracy = net.SGD(
training_data[:training_set_size],
num_epoches,
10,
0.5,
evaluation_data=test_data,
lmbda=lmbda,
monitor_evaluation_accuracy=True,
monitor_evaluation_cost=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
with open(filename, "w") as f:
json.dump([test_cost, test_accuracy, training_cost, training_accuracy],
f)
def run_networks():
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
training_data, validation_data, test_data = list(training_data), list(
validation_data), list(test_data)
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
accuracies = []
for size in SIZES:
print("\n\nTraining network with data set size %s" % size)
net.large_weight_initializer()
num_epochs = 1500000 // size
net.SGD(training_data=training_data[:size],
epochs=num_epochs,
mini_batch_size=10,
eta=0.5,
lmbda=size * 0.0001,
full_batch=True)
accuracy = net.accuracy(
validation_data) / 100.0 # Because val has 10,000
print("Accuracy was %s percent" % accuracy)
accuracies.append(accuracy)
f = open("more_data.json", "w")
json.dump(accuracies, f)
f.close()
def getNeuralNetFromUser():
neural_net_file = "resources/neural_net" #JSON in a text file used to load the neural network
net = None
print "Load Neural Network from file?"
value = getInput("-1 for training a new network, other key to load a trained one: ")
if (value == '-1'):
net_layers = [TOTAL_SIZE] #List of neurons, input layer == N pixels
i = 1
print "For each layer, insert the number of neurons\nInsert -1 to finish: "
while(True):
s_layer = "Layer {}: ".format(i)
layer = int(getInput(s_layer))
if(layer == -1):
break
net_layers.append(layer)
i += 1
net_layers.append(N_ELEMENTS) #Output layer == N possible output values
net = network2.Network(net_layers, cost=network2.CrossEntropyCost)
net.large_weight_initializer()
else:
value = getInput("-1 for specifying the neural network file. Other to load the default '{}': ".format(neural_net_file))
if(value == '-1'):
neural_net_file = getInput("Insert file name of the neural net to be loaded: ")
while(True):
if (isfile(neural_net_file)):
break
neural_net_file = getInput("Insert file name of the neural net to be loaded: ")
print "Name invalid, please try again"
net = network2.load(neural_net_file) #Loads an existing neural network from a file
return net
def main():
''' These settings achieved 95% accuracy on my particular hardware setup.
'''
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network2.Network([784, 30, 10])
net.SGD(training_data, 30, 15, 3) #, test_data=test_data)
''' And these settings achieved 97% accuracy, but took about
def test22():
net = network2.Network([784, 100, 10], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
net.SGD(training_data,
30,
10,
0.5,
evaluation_data=test_data,
monitor_evaluation_accuracy=True)
def get_let_from_2nd_nn_ceg(letter):
net = network2.Network([1024, 30, 3], cost=network2.CrossEntropyCost)
biases_saved = np.load('nn_two_stage/biases_ceg.npy')
weights_saved = np.load('nn_two_stage/weights_ceg.npy')
output = np.argmax(net.feedforward(letter, biases_saved = biases_saved, weights_saved = weights_saved))
return get_eq_let_ceg(output)
def main():
global exit_clicked
exit_clicked = False
plt.ioff()
plt.show()
global mode
mode = 0
if len(sys.argv) > 1:
if sys.argv[1] == '1': mode = 1
if sys.argv[1] == '2': mode = 2
fig, axs = plt.subplots(2, 2)
if mode == 1:
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
#train ('trainednet2')
fig.canvas.draw()
img = np.zeros((28, 28))
net = network2.load(
'C:/Users/ThomasReichert/source/repos/TheLIFO/NeuralNetMINT/trainednet2'
)
myobj = drawableIMG(fig, img, net)
if mode == 2:
img = 0.5 * np.ones((140, 140))
net = network2.Network([2, 10, 10, 2], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
myobj = NNIMG(fig, img, net)
plt.draw()
plt.ion()
while not (exit_clicked):
plt.pause(0.001)
#obj = axs[1, 1].imshow(np.reshape(test_data[0][0],(28,28)),vmin=0, vmax=1)
#for i in range(1,10):
# obj.set_data(np.reshape(test_data[i][0],(28,28)))
# x = net1.feedforward(test_data[i][0])
# x = np.argmax(x)
# result ="result: " + str(x)
# print 'result:', x
# plt.suptitle(result)
# fig.canvas.draw()
# wait()
print "exit"
def test_cross_entropy_cost():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net2 = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
net2.large_weight_initializer()
net2.SGD(training_data, 30, 10, 0.5, evaluation_data=test_data, monitor_evaluation_accuracy=True)
# import network
# from mnist_load import MnistLoad
# loader = MnistLoad('../data-new/')
# train_data, test_data = loader.loadAsNumpyData()
# net = network.Network([784, 30, 10])
# net.SGD(train_data, 30, 10, 3.0, test_data = test_data)
def main2():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network2.Network([784, 100, 10], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
net.SGD(training_data,
30,
10,
0.5,
evaluation_data=test_data,
lmbda=5.0,
monitor_evaluation_accuracy=True,
monitor_training_accuracy=True)
def call_network2():
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
net.SGD(training_data,
30,
10,
0.5,
lmbda=5.0,
p_evaluation_data=validation_data,
L1_regularization=True,
monitor_evaluation_accuracy=True,
monitor_evaluation_cost=True,
monitor_training_accuracy=True,
monitor_training_cost=True)
def __init__(self, env):
# extract relevant environment data
self.x_limit = env.x_limit
self.y_limit = env.y_limit
self.action_space = env.action_space
# the input size determines the network architecture
self.input_size = 2 * self.x_limit * self.y_limit + len(
self.action_space)
# define the network
self.network = network2.Network([self.input_size, 5, 1],
cost=network2.CrossEntropyCost)
# initialize the network weights
self.network.default_weight_initializer()
def get_let_from_2nd_nn_ijltIL1(letter):
net = network2.Network([1024, 30, 7], cost=network2.CrossEntropyCost)
biases_saved = np.load('nn_two_stage/biases_ijltIL1.npy',
encoding='latin1')
weights_saved = np.load('nn_two_stage/weights_ijltIL1.npy',
encoding='latin1')
output = np.argmax(
net.feedforward(letter,
biases_saved=biases_saved,
weights_saved=weights_saved))
return get_eq_let_ijl1I(output)
def test_sgd(self):
net_3 = network2.Network([784, 10, 10])
data_wrapper = mnist_loader.load_data_wrapper()
train = data_wrapper[0][:1000] # 1000条训练数据
eval_data = data_wrapper[1][:300] # 验证集的前2000条数据
sgd = net_3.SGD(train,
10,
10,
0.01,
lmbda=0.0,
evaluation_data=eval_data,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
print sgd # 输出训练集和验证集的cost和accuracy
def main_02():
t1 = time.gmtime(time.time())
print('training NN2 running....%d:%d:%d' %
(t1.tm_hour, t1.tm_min, t1.tm_sec))
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
net.SGD(training_data,
30,
10,
0.5,
evaluation_data=test_data,
monitor_evaluation_accuracy=True)
t1 = time.gmtime(time.time())
print('training NN2 running....%d:%d:%d end' %
(t1.tm_hour, t1.tm_min, t1.tm_sec))
