def run_network(filename, n, eta):
"""Train the network using both the default and the large starting
weights. Store the results in the file with name ``filename``,
where they can later be used by ``make_plots``.
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784, n, 10], cost=network2.CrossEntropyCost)
print "Train the network using the default starting weights."
default_vc, default_va, default_tc, default_ta = net.SGD(
training_data, 30, 10, eta, lmbda=5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True
)
print "Train the network using the large starting weights."
net.large_weight_initializer()
large_vc, large_va, large_tc, large_ta = net.SGD(
training_data, 30, 10, eta, lmbda=5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True
)
f = open(filename, "w")
json.dump(
{
"default_weight_initialization": [default_vc, default_va, default_tc, default_ta],
"large_weight_initialization": [large_vc, large_va, large_tc, large_ta],
},
f,
)
f.close()
def trainnetwork():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
network= Network([1200,30,10])
network.SGD(training_data,30,10,.5,test_data=test_data)
network.save("network.txt")
def train_mnist_worker(params):
net_id = params.get('net-id', 'nn')
layers = [784]
layers.extend([int(i) for i in params.get('layers', [15])])
layers.append(10)
net_params = {}
net_params['epochs'] = int(params.get('epochs', 1))
net_params['mini_batch_size'] = int(params.get('mini-batch-size', 4))
net_params['eta'] = float(params.get('eta', 0.1))
net_params['lmbda'] = float(params.get('lmbda', 0.0001))
net_params['layers'] = layers
redis.set(redis_key('params', net_id), json.dumps(net_params))
redis.set(redis_key('status', net_id), 'train_mnist: started')
net = Network(layers)
training_data, validation_data, test_data = load_data_wrapper()
redis.set(redis_key('status', net_id), 'train_mnist: training with mnist data')
net.SGD(training_data, net_params['epochs'],
net_params['mini_batch_size'],
net_params['eta'],
net_params['lmbda'])
redis.set(redis_key('data', net_id), net.tostring())
redis.set(redis_key('status', net_id), 'train_mnist: trained')
def __init__(self, fileNames=[]):
self.fileNames = fileNames
self.trainingData,self.valData,self.testData = \
mnist_loader.load_data_wrapper()
self.data = [self.trainingData, self.valData, self.testData]
self.fileNameDataPairs = zip(self.fileNames, self.data)
def visualize():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# Unzipping gives tuples, but we want arrays of values.
training_input = [x.transpose()[0] for x in zip(*training_data)[0]]
test_input = [x.transpose()[0] for x in zip(*test_data)[0]]
# Get the y values.
test_target = [y for y in zip(*test_data)[1]]
# Apply SVD to the training input.
u, s, v = np.linalg.svd(training_input, full_matrices=False)
print u.shape
print s.shape
print v.shape
print "Generating embeddings..."
#print v[0]
print v[0].shape
embeddings = [np.dot(test_inp, np.transpose(v[:10][:])) for test_inp in test_input]
print embeddings[0].shape
# Do dimensionality reduction into 2 dimensions.
print "Performing dimensionality reduction using t-sne..."
tsne = TSNE()
reduced_vecs = tsne.fit_transform(embeddings)
print reduced_vecs[0]
# Graph all of the points, where points corresponding to the same digit will have the same color.
colors = ['r', 'b', 'g', 'c', 'm', 'k', 'y', (.2, .2, .2), (.4, 0, .5), (.8, .2, 0)]
red_patch = mpatches.Patch(color='red', label='1')
patches = [mpatches.Patch(color=colors[i], label='%i'% i) for i in range(len(colors))]
plt.legend(handles=patches)
for i in range(len(reduced_vecs)):
plt.plot([reduced_vecs[i][0]], [reduced_vecs[i][1]], 'o', color=colors[test_target[i]])
plt.show()
def main():
# Load the data
full_td, _, _ = mnist_loader.load_data_wrapper()
td = full_td[:1000] # Just use the first 1000 items of training data
epochs = 500 # Number of epochs to train for
print "\nTwo hidden layers:"
net = network2.Network([784, 30, 30, 10])
initial_norms(td, net)
abbreviated_gradient = [ag[:6] for ag in get_average_gradient(net, td)[:-1]]
print "Saving the averaged gradient for the top six neurons in each " + "layer.\nWARNING: This will affect the look of the book, so be " + "sure to check the\nrelevant material (early chapter 5)."
f = open("initial_gradient.json", "w")
json.dump(abbreviated_gradient, f)
f.close()
shutil.copy("initial_gradient.json", "../../js/initial_gradient.json")
training(td, net, epochs, "norms_during_training_2_layers.json")
plot_training(epochs, "norms_during_training_2_layers.json", 2)
print "\nThree hidden layers:"
net = network2.Network([784, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_3_layers.json")
plot_training(epochs, "norms_during_training_3_layers.json", 3)
print "\nFour hidden layers:"
net = network2.Network([784, 30, 30, 30, 30, 10])
initial_norms(td, net)
training(td, net, epochs, "norms_during_training_4_layers.json")
plot_training(epochs, "norms_during_training_4_layers.json", 4)
def convert_test():
fulltrain = pd.read_csv('dat/train.csv')
trainx = fulltrain.drop(['Id','Cover_Type'], axis=1) # Features
trainy = fulltrain['Cover_Type'] # Target
newtrain = train_pd_to_nielsen(trainx, trainy, trainy.max())
print np.asarray(newtrain).shape
print np.asarray(newtrain)[0][0].shape
print np.asarray(newtrain)[0][1].shape
nisttrain, nistvalid, nisttest = mnist_loader.load_data_wrapper(nielsen_path + 'data/mnist.pkl.gz')
print np.asarray(nisttrain).shape
print np.asarray(nisttrain)[0][0].shape
print np.asarray(nisttrain)[0][1].shape
print '-'*50
newtest = test_pd_to_nielsen(trainx, trainy)
print np.asarray(newtest).shape
print np.asarray(newtest)[0][0].shape
print np.asarray(newtest)[0][1].shape
print np.asarray(nistvalid).shape
print np.asarray(nistvalid)[0][0].shape
print np.asarray(nistvalid)[0][1].shape
def run():
np.random.seed(0)
data = load_data_wrapper()
initial_learning_rate = 0.2
network = Network(
[784, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 10],
activations_function=ReLU(),
cost=CrossEntropy(),
stopping_criteria=NEpochs(50), # LearningRateDecreaseLimit(
# initial_learning_rate=initial_learning_rate,
# limit=1/2
# ),
learning_rate=FixedLearningRate(0.01), # HalfLRIfNoDecreaseInNEpochs(
# monitor_parameter='validation_cost',
# max_epochs=1,
# initial_learning_rate=initial_learning_rate
# ),
update_algorithm=Momentum(momentum=0.5, base_algorithm=L2UpdateAlgorithm(lmbda=5)),
weight_initializer=initialize_input_dim_normalized_weights,
)
t0 = datetime.utcnow()
network.sgd(training_data=data[0], validation_data=data[1], test_data=data[2], mini_batch_size=12)
print("Total time fast {}".format(datetime.utcnow() - t0))
plot_stats(network)
return network.log
def main():
f = open('myNeural.txt', 'r')
net = pickle.load(f)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# 处理为 scikit_learn所需的数据结构
X_train = [np.reshape(x, (1, -1))[0] for (x, y) in training_data]
y_train = [np.argmax(np.reshape(y, (1, -1))[0]) for (x, y) in training_data]
# Fit estimators
ESTIMATORS = {
# KNN
"K-nn": neighbors.KNeighborsClassifier().fit(X_train, y_train),
# 朴素贝叶斯
"native-bayes": BernoulliNB().fit(X_train, y_train)
# 决策树
# 聚类
}
for i in test_data:
print '=================================='
testdata = i[0]
print '正确的结果为%d' % i[1]
# 使用神经元网络验证
result = np.argmax(net.feedforward(testdata))
print '使用神经元网络分类的结果为', result
for name, estimator in ESTIMATORS.items():
print '使用%s进行分类，分类结果为%s' % (name, estimator.predict(np.reshape(testdata, (1, -1))))
def run_network(filename, num_epochs, training_set_size=1000, lmbda=0.0):
"""Train the network for ``num_epochs`` on ``training_set_size``
images, and store the results in ``filename``. Those results can
later be used by ``make_plots``. Note that the results are stored
to disk in large part because it's convenient not to have to
``run_network`` each time we want to make a plot (it's slow).
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
#net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
net = network2.Network([784, 30, 10], cost=network2.LogLikelihoodCost)
net.large_weight_initializer()
test_cost, test_accuracy, training_cost, training_accuracy, output_activations \
= net.SGD(training_data[:training_set_size], num_epochs, 10, 0.5,
evaluation_data=test_data, lmbda = lmbda,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
f = open(filename, "w")
'''
print "test_cost",type(test_cost),test_cost
print
print "test_accuracy",type(test_accuracy), test_accuracy
print
print "training_cost",type(training_cost), training_cost
print
print "training_accuracy",type(training_accuracy), training_cost
print
'''
json.dump([test_cost, test_accuracy, training_cost, training_accuracy, output_activations], f)
f.close()
def run_network(filename):
"""Train the network using both the default and the large starting
weights. Store the results in the file with name ``filename``,
where they can later be used by ``make_plots``.
"""
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost())
print "Train the network using the default starting weights."
default_vc, default_va, default_tc, default_ta \
= net.SGD(training_data, 30, 10, 0.01,
evaluation_data=validation_data, lmbda = 0.001,
monitor_evaluation_accuracy=True)
print "Train the network using the large starting weights."
net.large_weight_initializer()
large_vc, large_va, large_tc, large_ta \
= net.SGD(training_data, 30, 10, 0.01,
evaluation_data=validation_data, lmbda = 0.001,
monitor_evaluation_accuracy=True)
f = open(filename, "w")
json.dump({"default_weight_initialization":
[default_vc, default_va, default_tc, default_ta],
"large_weight_initialization":
[large_vc, large_va, large_tc, large_ta]},
f)
f.close()
def run_networks():
"""Train networks using three different values for the learning rate,
and store the cost curves in the file ``multiple_eta.json``, where
they can later be used by ``make_plot``.
"""
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
results = []
for eta in LEARNING_RATES:
print "\nTrain a network using eta = " + str(eta)
net = network2.Network([784, 30, 10])
results.append(
net.SGD(
training_data,
NUM_EPOCHS,
10,
eta,
lmbda=5.0,
evaluation_data=validation_data,
monitor_training_cost=True,
)
)
f = open("multiple_eta.json", "w")
json.dump(results, f)
f.close()
def load_data():
train, val, test = mnist_loader.load_data_wrapper()
train = list(train)
val = list(val)
test = list(test)
return train, val, test
def learn(request):
tr_d, v_d, t_d = mnist_loader.load_data_wrapper()
net.SGD(tr_d,30,10,3.0)
print net.evaluate(t_d)
return HttpResponseRedirect('/')
def __init__(self, sizes):
print("Starting...");
start = time.time()
training_data, validation_data, test_data = \
mnist_loader.load_data_wrapper()
self.net = network.Network(sizes)
(self.net).SGD(training_data, 30, 10, 3.0, test_data=test_data)
end = time.time()
print("Done!\nTime Taken: " + str(end - start));
def recognize(img_name):
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
img_list= loadImage(img_name)
net = network.Network([784, 100, 10]) #input,hidden,output
#net.SGD(training_data, 5, 10, 2.0, test_data = test_data)
f=open('100hl.bin','rb')
net.biases = np.load(f)
net.weights = np.load(f) # load trained weights and biases
f.close() #close the file after reading weights and biases
#check for image
return net.feedforward(img_list)
def main():
# Time to processs dataset
start_process = timeit.default_timer()
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = NeuralNetwork([784, 30, 10])
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
time_neural = timeit.default_timer() - start_process
print "The total time to run the neural network was: %d seconds" %(int(time_neural))
def run_networks():
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# instantiate network
net = nnetwork.Network([INPUT_NEURONS,HIDDEN_NEURONS,OUTPUT_NEURONS])
# run SGD
results = net.gradientDescent(training_data, BATCH_SIZE, eta, NUM_EPOCHS,
test_data=test_data)
f = open("learning.json", "w")
json.dump(results, f)
f.close()
def test_cross_entropy_cost():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net2 = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
net2.large_weight_initializer()
net2.SGD(training_data, 30, 10, 0.5, evaluation_data=test_data, monitor_evaluation_accuracy=True)
# import network
# from mnist_load import MnistLoad
# loader = MnistLoad('../data-new/')
# train_data, test_data = loader.loadAsNumpyData()
# net = network.Network([784, 30, 10])
# net.SGD(train_data, 30, 10, 3.0, test_data = test_data)
def main():
filename = 'test'
# load data
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# example
epochs = 5
net = network2.Network([784, 10], cost=network2.CrossEntropyCost)
vc, va, tc, ta = net.SGD(training_data=training_data, epochs=epochs, mini_batch_size=100, eta=0.1, lmbda = 0.1, reg = 2,
evaluation_data=validation_data,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
def run_networks():
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = nnetwork1.Network([INPUT_NEURONS,HIDDEN_NEURONS,OUTPUT_NEURONS], mu)
results = []
for eta in LEARNING_RATES:
print "\nTrain a network using eta = "+str(eta)
results.append(net.gradientDescent(training_data, batch_size, eta, NUM_EPOCHS,
test_data=test_data))
f = open("eta_graph1.json", "w")
json.dump(results, f)
f.close()
def run_networks():
# Make results more easily reproducible
random.seed(12345678)
np.random.seed(12345678)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = nnetwork2.Network([784, 30, 10], 0.5)
results = []
for eta in LEARNING_RATES:
print "\nTrain a network using eta = "+str(eta)
results.append(net.gradientDescent(training_data, 10, eta, NUM_EPOCHS,
test_data=test_data))
f = open("eta_graph2.json", "w")
json.dump(results, f)
f.close()
def __init__(self):
QtGui.QMainWindow.__init__(self)
Ui_MainWindow.__init__(self)
self.setupUi(self)
self.trainButton.clicked.connect(self.trainButtonAction)
self.loadParamButton.clicked.connect(self.loadData)
self.saveParamButton.clicked.connect(self.saveData)
self.testDigitButton.clicked.connect(self.testDigit)
self.showDigitButton.clicked.connect(self.showDigit)
self.psDigitTest.clicked.connect(self.testPsDigit)
self.drawButton.clicked.connect(self.drawDigitAndTest)
self.training_data, self.validation_data, self.test_data = mnist_loader.load_data_wrapper()
self.net = network.Network([784, 30, 10])
self.textEdit.append('Network created.')
def main(layers=None):
# Load the data
full_td, _, _ = mnist_loader.load_data_wrapper()
td = full_td[:1000] # Just use the first 1000 items of training data
epochs = 500 # Number of epochs to train for
if layers is None:
layers = [784, 30, 30, 10]
net = network.Network(layers)
initial_norms(td, net)
abbreviated_gradient = [
ag[:6] for ag in get_average_gradient(net, td)[:-1]]
training(td, net, epochs, "norms_during_training_2_layers.json")
plot_training(
epochs, "norms_during_training_2_layers.json", 2)
def run_networks():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost())
accuracies = []
for size in SIZES:
print "\n\nTraining network with data set size %s" % size
net.large_weight_initializer()
num_epochs = 1500000 / size
net.SGD(training_data[:size], num_epochs, 10, 0.05, lmbda = 0.001)
accuracy = net.accuracy(validation_data) / 100.0
print "Accuracy was %s percent" % accuracy
accuracies.append(accuracy)
f = open("more_data.json", "w")
json.dump(accuracies, f)
f.close()
def run_network(filename, lmbda=0.0):
"""Train the network, and store the results in ``filename``. Those
results can later be used by ``make_plots``. Note that the
results are stored to disk in large part because it's convenient
not to have to ``run_network`` each time we want to make a plot
(it's slow).
"""
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost())
test_cost, test_accuracy, training_cost, training_accuracy \
= net.SGD(training_data[:1000], NUM_EPOCHS, 10, 0.05,
evaluation_data=test_data, lmbda = lmbda,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True)
f = open(filename, "w")
json.dump([test_cost, test_accuracy, training_cost, training_accuracy], f)
f.close()
def visualize():
infile = open("trained_autoencoder.pkl")
auto_enc = pickle.load(infile)
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# Unzipping gives tuples, but we want arrays of values.
training_input = [x for x in zip(*training_data)[0]]
test_input = [x for x in zip(*test_data)[0]]
# Get the y values.
test_target = [y for y in zip(*test_data)[1]]
print test_target[0]
'''
# Encode all of the MNIST test set using the autoencoder.
# TODO: get rid of debugging, do all points not just 50
print "Encoding MNIST using autoencoder..."
autoencoder_encoded_vecs = [auto_enc.feedforward(test_inp, embed=True).transpose()[0] for test_inp in test_input]
print len(autoencoder_encoded_vecs)
# print autoencoder_encoded_vecs[0]
# print autoencoder_encoded_vecs[0].shape
'''
# Do dimensionality reduction into 2 dimensions using t-sne.
print "Performing dimensionality reduction using t-sne..."
tsne = sklearn.manifold.TSNE()
# Try just using tsne on the raw MNIST digits data.
autoencoder_encoded_vecs = [inp.transpose()[0] for inp in test_input]
reduced_vecs = tsne.fit_transform(autoencoder_encoded_vecs)
print reduced_vecs[0]
#plt.plot([p[0] for p in reduced_vecs[:30]], [p[1] for p in reduced_vecs[:30]], 'ro')
# Graph all of the points, where points corresponding to the same digit will have the same color.
colors = ['r', 'b', 'g', 'c', 'm', 'k', 'y', (.2, .2, .2), (.4, 0, .5), (.8, .2, 0)]
red_patch = mpatches.Patch(color='red', label='1')
patches = [mpatches.Patch(color=colors[i], label='%i'% i) for i in range(len(colors))]
plt.legend(handles=patches)
for i in range(len(reduced_vecs)):
plt.plot([reduced_vecs[i][0]], [reduced_vecs[i][1]], 'o', color=colors[test_target[i]])
plt.show()
def main():
# Load MNIST
# training_input = mnist_loader.load_training_input()
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# Unzipping gives tuples, but we want arrays of values.
training_input = [x for x in zip(*training_data)[0]]
test_input = [x for x in zip(*test_data)[0]]
print type(training_input[0])
#print(training_input[0][:10])
#print(autoencoder.add_noise(training_input[0])[:10])
# Make autoencoder network
input_size = len(training_input[0])
hidden_size = 100
auto_enc = autoencoder.Autoencoder(input_size, hidden_size)
#auto_enc.sgd(training_input, 5, 100, 3.0)
auto_enc.sgd(training_input, 5, 100, 3.0, test_input)
# Save the trained autoencoder to file.
outfile = open("trained_autoencoder.pkl", "w")
pickle.dump(auto_enc, outfile)
mse_error = auto_enc.test(test_input)
print "mse_error: ", mse_error
def Main():
#Load MNIST ****************************************************************
#mnist.pkl.gz(50000） Accuracy 0.96
#mnist_expanded.pkl.gz(250000） Accuracy 0.97
fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
# *********************************************************************
path = "..\\TmpLogs\\"
if not os.path.isdir(path):
os.mkdir(path)
fnNetworkData = "{}{}_NetData".format(path, rn.RvNeuralNetwork.__name__)
fnSaved = ""
#Hyper pameters -------------------------------------------
loop = 10 # loop effect，10, 30 all above 0.95
stepNum = 10 # stepNum effect, 10->0.9, 100->0.5
learnRate = 0.1 # learnRate and lmbda will affect each other
lmbda = 5.0 #add lmbda(Regularization) to solve overfitting
# Training ***********************************************************
# Ask DoTraining-
DoTraining = ri.Ask_YesNo("Execute Training?", "y")
if DoTraining:
"""
[784,50,10],
loop=10, 0.9695
loop=100, 0.9725
"""
# 建立 RvNeuralNetWork----------------------------------------------
inputNeusNum = len(lstTrain[0][0])
lyr1NeuNum = 50
lyr2NeuNum = len(lstTrain[0][1])
lyrsNeus = [inputNeusNum, lyr1NeuNum]
lyrsNeus = ri.Ask_Add_Array_Int("Input new layer Neurons num.",
lyrsNeus, lyr1NeuNum)
lyrsNeus.append(lyr2NeuNum)
#net = rn.RvNeuralNetwork( \
# rn.RvNeuralNetwork.LayersNeurons_To_RvNeuralLayers(lyrsNeus))
#net = rn.RvNeuralNetwork.Class_Create_LayersNeurons(lyrsNeus)
#net = rn.RvNeuralNetwork(lyrsNeus) # ([784,50,10])
cnvLyrSId = 0
lyrObjs = []
# Add RvConvolutionLayer--------------------------------
EnableCovolutionLayer = ri.Ask_YesNo("Add ConvolutionLayer?", "y")
if EnableCovolutionLayer:
lyrsNeus, cnvLyrSId, cnvLyr = Add_ConvLayer(
lyrObjs, lyrsNeus, inputNeusNum, cnvLyrSId)
# Create Layer Object array -------------------------------
for iLyr in range(cnvLyrSId, len(lyrsNeus) - 1):
lyrObjs.append(
rn.RvNeuralLayer([lyrsNeus[iLyr], lyrsNeus[iLyr + 1]]))
net = rn.RvNeuralNetwork(lyrObjs)
# Ask nmtivation ------------------_----------
enumActivation = ri.Ask_Enum("Select Activation method.",
nm.EnumActivation,
nm.EnumActivation.afReLU)
for lyr in net.NeuralLayers:
lyr.ClassActivation, lyr.ClassCost = \
nm.Get_ClassActivation(enumActivation)
net.Motoring_TrainningProcess = rn.Debug
net.NetEnableDropOut = ri.Ask_YesNo("Execute DropOut?", "n")
if net.NetEnableDropOut:
enumDropOut = ri.Ask_Enum("Select DropOut Method.",
nm.EnumDropOutMethod,
drpOut.eoSmallActivation)
rn.gDropOutRatio = ri.Ask_Input_Float("Input DropOut ratio.",
rn.gDropOutRatio)
net.Set_DropOutMethod(enumDropOut, rn.gDropOutRatio)
monitoring = ri.Ask_YesNo("Watch training process?", "y")
net.Motoring_TrainningProcess = monitoring
# Caculate proper hyper pameters ---
DoEvaluate_ProperParams = ri.Ask_YesNo(
"Auto-caculating proper hyper pameters?", "n")
if DoEvaluate_ProperParams:
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_lmbda(
net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum,
learnRate, lmbda)
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_learnRate(
net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum,
learnRate, lmbda)
else:
loop, stepNum, learnRate, lmbda = rf.Ask_Input_SGD(
loop, stepNum, learnRate, lmbda)
print(
"Hyper pameters: Loop({}), stepNum({}), learnRatio({}), lmbda({})\n"
.format(loop, stepNum, learnRate, lmbda))
start = time.time()
# 開始網路訓練-
net.Train(lstTrain, loop, stepNum, learnRate, lstV, lmbda)
dT = time.time() - start
fnSaved = rf.Save_NetworkDataFile(net, fnNetworkData, loop, stepNum,
learnRate, lmbda, dT)
# Ask DoPredict----------------------------------------------------
DoPredict = True
if DoPredict:
if (os.path.isfile(fnSaved)):
fn1 = fnSaved
else:
fn1 = ".\\{}_NetData_DontDelete.txt".format(
rn.RvNeuralNetwork.__name__)
rn.Debug_Plot = True #ri.Ask_YesNo("Plot Digits?", "n")
# Predict_Digits(net, lstT)
pltFn.Predict_Digits_FromNetworkFile(fn1, lstT, rn.Debug_Plot)
import mnist_loader
from NeuralNetwork import Network
train_data, validate_data, test_data = mnist_loader.load_data_wrapper()
net = Network(configurationFileName='NetConfig')
print('Done {0} from {1}'.format(net.check(test_data), len(test_data)))
import sys
def test_train(self):
train_data, val_data, test_data = loader.load_data_wrapper()
sizes = [784, 30, 30, 10]
model = nn.Network(sizes)
model.train(train_data, 30, 3.0, 25, test_data)
def main():
dimension_string = sys.argv[1]
dimensions = map(int, dimension_string.strip('[]').split(','))
net = NeuralNetwork(dimensions) # map(int,input.strip('[]').split(','))
(training_data, validation, test_data) = load_data_wrapper()
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
import mnist_loader import tensorflow as tf import math import numpy as np import matplotlib.pyplot as plt x_training,y_training,x_validation,y_validation,x_test,y_test = mnist_loader.load_data_wrapper() N_training=len(x_training) N_validation=len(x_validation) N_test=len(x_test) N_epochs = 5 learning_rate = 3.0 batch_size = 10 N1 = 784 #equals N_inputs N2 = 30 N3 = 30 N4 = 30 N5 = 10 N_in=N1 N_out=N5
import mnist_loader,DigitNN training_data, validation_data, test_data = \ mnist_loader.load_data_wrapper() net = DigitNN.NeuralNetwork([784, 30, 10]) net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
def Main():
#Hyper pameters -------------------------------------------
loop = 5 # loop effect，10, 30 all above 0.95
stepNum = 10 # stepNum effect, 10->0.9, 100->0.5
learnRate = 0.1 # learnRate and lmbda will affect each other
lmbda = 5.0 #add lmbda(Regularization) to solve overfitting
dropOutRatio = rn.gDropOutRatio
# Load net file and continune training--
fns, fn0s = rfi.Get_FilesInFolder(".\\NetData\\", [".cnn", ".dnn"])
aId = ri.Ask_SelectItem("Select network file", fn0s, 0)
fn1 = fns[aId]
#
# print(rfi.ExtractFilePath(fn1))
# print(rfi.ExtractFileName(fn1))
# print(rfi.ExtractFileExt(fn1))
# return
if (os.path.isfile(fn1)):
net = rn.RvNeuralNetwork(fn1)
enumDropOut = net.NetEnumDropOut.value
dropOutRatio = net.NetDropOutRatio
#'BestAccuracyRatio' : net.BestAccuracyRatio
#loop = net.Train_Loop
learnRate = net.Train_LearnRate
lmbda = net.Train_Lmbda
else:
net = None
if (None != net):
#Load MNIST ****************************************************************
#mnist.pkl.gz(50000） Accuracy 0.96
#mnist_expanded.pkl.gz(250000） Accuracy 0.97
fn = "..\\data\\mnist.pkl.gz" # "..\\data\\mnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
sYN = "y" if net.NetEnableDropOut else "n"
enableDropOut = ri.Ask_YesNo("Excute DropOut?", sYN)
if enableDropOut:
enumDropOut = ri.Ask_Enum("Select DropOut Method.",
nm.EnumDropOutMethod,
drpOut.eoSmallActivation)
rn.gDropOutRatio = ri.Ask_Input_Float("Input DropOut ratio.",
dropOutRatio)
if enableDropOut:
net.Set_DropOutMethod(enumDropOut, rn.gDropOutRatio)
#Auto caculate proper Hyper pameters ---
loop, stepNum, learnRate, lmbda = rf.Ask_Input_SGD(
loop, stepNum, learnRate, lmbda)
print(
"Hyper Pameters: Loop({}), stepNum({}), learnRatio({}), lmbda({})\n"
.format(loop, stepNum, learnRate, lmbda))
DoKeepTraining = True
while (DoKeepTraining):
start = time.time()
# Start Training
net.Train(lstTrain,
loop,
stepNum,
learnRate,
lstV,
lmbda,
blInitialWeiBias=False)
dT = time.time() - start
if net.BestAccuracyRatio > net.Get_NetworkFileData(fn1):
rf.Save_NetworkDataFile(net, fn1, loop, stepNum, learnRate,
lmbda, dT)
print("Save Network file \"{}\"".format(fn1))
DoKeepTraining = ri.Ask_YesNo("Continue training?", "y")
KL_loss = tf.reduce_mean(KL_loss)
total_loss = recon_loss + KL_loss
train_op = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(total_loss)
losses = {
'recon_loss': recon_loss,
'total_loss': total_loss,
'KL_loss': KL_loss,
}
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, './VAE_mnist.ckpt')
training_data = mnist_loader.load_data_wrapper()
random.shuffle(training_data)
a = tf.placeholder(tf.float32, [50000, 784])
dataset = tf.data.Dataset.from_tensor_slices(a)
dataset = dataset.prefetch(buffer_size=1000)
dataset = dataset.batch(batch_size)
iterator = dataset.make_initializable_iterator()
next = iterator.get_next()
# sess.run(tf.global_variables_initializer())
print("session initialized 1")
sess.run(iterator.initializer, feed_dict={a: training_data})
print("iterator initialized ")
# for i in range(5000):
# # print(i)
# if i>0 and i % (50000 // batch_size) == 0:
#!/usr/bin/env/python3.5 import mnist_loader import network2 as net import numpy as np import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from timeit import default_timer # read the data TRAINING_DATA, VALIDATION_DATA, TEST_DATA = mnist_loader.load_data_wrapper() TRAINING_DATA, VALIDATION_DATA, TEST_DATA = list(TRAINING_DATA), list(VALIDATION_DATA), list(TEST_DATA) # size of data sets TRAINING_SIZE, TEST_SIZE = len(TRAINING_DATA), len(TEST_DATA) # smaller subset for faster tuning N_VAL_SAMPLES = 1000 SMALL_VALIDATION_DATA = VALIDATION_DATA[:N_VAL_SAMPLES] SMALL_TRAINING_DATA = TRAINING_DATA[:5000] # initialize constants LAYERS = [784, 30, 10] # training parameters TRAINING_EPOCHS, TEST_EPOCHS = 30, 100 # initial grid search values INITIAL_LMBDA = 0 # used for LR search INITIAL_LEARNING_RATES = np.logspace(-3, 3, 7)
def main():
""""""
train, val, test = load_data_wrapper()
model = Network([784, 60, 10])
model.SGD(train, 30, 10, 3, val)
print('Evaluation on test: {0} / {1}'.format(model.evaluate(test), len(test)))
"""
mnist_sgd
~~~~~~~~~
A classifier program for recognizing handwritten digits from the MNIST
data set, using a neural network classifier with stochastic gradient descent."""
"""
Let's start by loading in the MNIST data using a little helper program, mnist_loader.py
"""
import mnist_loader
train, val, test = mnist_loader.load_data_wrapper()
# alternative code to read MNIST data
"""
import gzip, pickle
ff = gzip.open('../data/mnist.pkl.gz','rb')
u = pickle._Unpickler( ff )
u.encoding = 'latin1'
train, val, test = u.load()
ff.close()
"""
"""
After loading the MNIST data, we'll set up a Network with 30 hidden neurons
using the class ``network``
"""
import network
net = network.Network([784, 30, 10])
"""
Finally, we'll use stochastic gradient descent
to learn from the MNIST training data
over 30 epochs,
import mnist_loader as ml import network train_data, val_data, test_data = map(list, ml.load_data_wrapper()) print(len(train_data), len(test_data)) net = network.Network([784, 30, 10]) net.SGD(train_data, 30, 20, 0.1, test_data=test_data)
import os
import sys
sys.path.append("D:/Work/neural-networks-and-deep-learning/src")
import mnist_loader
import network2_1
import numpy as np
training_data, validation_data, test_data =\
map(list, mnist_loader.load_data_wrapper())
# No L1
net = network2_1.Network([784, 10])
net.SGD(training_data[:30000], 30, 10, 3)
print("=" * 20)
print("Accuracy unregularized: {} / {}\n".format(net.accuracy(validation_data),
len(validation_data)))
# L1 applied
print("=" * 20)
lmbdas = [0.0001, 0.01, 0.1, 0.5, 1, 2, 3, 4, 5, 7, 10]
print("Accuracy with L1: \n")
for lmbda in lmbdas:
net = network2_1.Network([784, 10])
net.SGD(training_data[:30000], 30, 10, 3, lmbda=lmbda, L1_ratio=1)
print("Lambda: {}, accuracy: {} / {}".format(lmbda,
net.accuracy(validation_data),
len(validation_data)))
""" REMARKS:
Below is an example of the output when trained
############################
# Module: cs5600_6600_f20_hw03.py
# Your name
# Your A#
############################
# from network import Network
from mnist_loader import load_data_wrapper
from ann import ann
import random
import pickle as cPickle
import pathlib
import numpy as np
# load training, validation, and testing MNIST data
train_d, valid_d, test_d = load_data_wrapper()
input_layer = 784
output_layer = 10
def train_1_hidden_layer_anns(lwr=10,
upr=50,
eta=0.25,
mini_batch_size=10,
num_epochs=10):
assert 100 >= upr > lwr >= 10
assert lwr % 10 == 0 and upr % 10 == 0 # divisible by 10
while lwr <= upr:
print(f"==== Training {input_layer}x{lwr}x{output_layer} ANN ======")
net = ann([input_layer, lwr, output_layer])
net.mini_batch_sgd(train_d, num_epochs, mini_batch_size, eta, test_d)
elif outputs[outputIndex] == 0 or outputs[outputIndex] == False:
boolean = False
else:
raise Exception("Unacceptable (non-boolean) output " +
str(outputs[xIndex]))
listOfClauses += Output(exampleID, xIndex, boolean,
valDict).getClauses()
return strSum([str(clause) for clause in listOfClauses])
output = open("nand.cnf", "w")
#data = [([0,0], [0]), ([0,1], [1]), ([1,0], [1]), ([1,1], [0])]
#output.write(makeSatInstance(data, 6))
def processRound(l):
for i in range(len(l)):
l[i] = [[round(x) for x in l[i][0]], l[i][1]]
return l
data = processRound(mnist_loader.load_data_wrapper()[0][:1])
print len(data)
print "data loaded"
output.write(makeSatInstance(data, 840))
#data = [([0,0], [1]), ([0,1], [0]), ([1,0], [0]), ([1,1], [0])]
#output.write(makeSatInstance(data, 3))
trainSize = 1000
testSize = 1000
#array separated by commas for multiple hidden layers
hiddenLay = [100]
#array separeted by commas for different conv sizes
'''convs = (10, 8)
#array separated by commas for differ step size during convolution. Values must match or crash/bad results
convstep = (3,1)
pool = 1'''
convs = (28, 12)
convstep = (1, 1)
pool = 2
learningRate = 1e-7
epochs = 10000
training_data_full, validation_data_full, test_data_full = mnist_loader.load_data_wrapper(
)
#print(len(training_data), len(test_data))
training_data_temp = training_data_full[:trainSize]
test_data_temp = training_data_full[:trainSize]
real_test_data = test_data_full[:testSize]
#filters = [[[-1, -1, -1],[0,0,0],[1,1,1]],[[-1, -1, -1],[0,0,0],[1,1,1]],[[-1, -1, -1],[0,0,0],[1,1,1]],[[-1, -1, -1],[0,0,0],[1,1,1]]]
'''filters = np.array([-1,-1,-1,0,0,0,1,1,1,
1,1,1,0,0,0,-1,-1,-1,
1,0,-1,1,0,-1,1,0,-1,
-1,0,1,-1,0,1,-1,0,1,
-1,-1,0,-1,0,1,0,1,1,
0,-1,-1,1,0,-1,1,1,0,
1,1,0,1,0,-1,0,-1,-1,
0,1,1,-1,0,1,-1,-1,0])'''
def __init__(self, learning_rate=5.0, epochs_time=10):
self.net = network.Network([784, 30, 10])
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
self.net.SGD(training_data, epochs_time, 10, learning_rate)
def Main():
#Load MNIST ****************************************************************
#Use mnist.pkl.gz(50000 data） Accuracy 0.96
#Use mnist_expanded.pkl.gz(250000 data） Accuracy 0.97
fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
path = "..\\TmpLogs\\"
if not os.path.isdir(path):
os.mkdir(path)
fnNetworkData = "{}{}_DNN_Digit".format(path, rn.RvNeuralNetwork.__name__)
fnNetworkData1 = ""
#Hyper pameters -------------------------------------------
loop = 10 # loop effect，10, 30 all above 0.95
stepNum = 10 # stepNum effect, 10->0.9, 100->0.5
learnRate = 0.1 # learnRate and lmbda will affect each other
lmbda = 5.0 #add lmbda(Regularization) to solve overfitting
sTrain = "y"
# Training ***********************************************************
LoadAndTrain = ri.Ask_YesNo("Load exist model and continue training?", "n")
if LoadAndTrain:
fns, shortFns = rfi.Get_FilesInFolder(".\\NetData\\", [".dnn"])
aId = ri.Ask_SelectItem("Select DNN Network file", shortFns, 0)
fn1 = fns[aId]
net = rn.RvNeuralNetwork(fn1)
sTrain = "n"
initialWeiBias = False
else:
"""
[784,50,10], loop=100, 0.9725
"""
# Create RvNeuralNetWork----------------------------------------------
inputNeusNum = len(lstTrain[0][0])
lyr1NeuNum = 50
lyr2NeuNum = len(lstTrain[0][1])
lyrsNeus = [inputNeusNum, lyr1NeuNum]
lyrsNeus = ri.Ask_Add_Array_Int("Input new layer Neurons num.",
lyrsNeus, lyr1NeuNum)
lyrsNeus.append(lyr2NeuNum)
#net = rn.RvNeuralNetwork( \
# rn.RvNeuralNetwork.LayersNeurons_To_RvNeuralLayers(lyrsNeus))
#net = rn.RvNeuralNetwork.Class_Create_LayersNeurons(lyrsNeus)
net = rn.RvNeuralNetwork(lyrsNeus) # ([784,50,10])
initialWeiBias = True
digitIdOnly = 5
digitIdOnly = ri.Ask_Input_Integer(
"Train which Digit (0~9, -1 = All Digits): ", digitIdOnly)
# Training ***********************************************************
DoTrain = ri.Ask_YesNo("Do Training?", sTrain)
if DoTrain:
fnNetworkData = "{}_{}Lyr".format(fnNetworkData, len(net.NeuralLayers))
net.DoPloatWeights = ri.Ask_YesNo("Plot Neurons Weights?", 'n')
# Ask nmtivation ------------------_----------
enumActivation = ri.Ask_Enum("Select Activation method.",
nm.EnumActivation,
nm.EnumActivation.afReLU)
for lyr in net.NeuralLayers:
lyr.ClassActivation, lyr.ClassCost = \
nm.Get_ClassActivation(enumActivation)
net.NetEnableDropOut = ri.Ask_YesNo("Execute DropOut?", "n")
if net.NetEnableDropOut:
enumDropOut = ri.Ask_Enum("Select DropOut Method.",
nm.EnumDropOutMethod,
drpOut.eoSmallActivation)
rn.gDropOutRatio = ri.Ask_Input_Float("Input DropOut ratio.",
rn.gDropOutRatio)
net.Set_DropOutMethod(enumDropOut, rn.gDropOutRatio)
# Auto-Caculate proper hyper pameters ---
DoEvaluate_ProperParams = ri.Ask_YesNo(
"Auto-Caculating proper hyper pameters?", "n")
if DoEvaluate_ProperParams:
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_lmbda(
net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum,
learnRate, lmbda)
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_learnRate(
net, net.Train, lstTrain[:1000], lstV[:500], loop, stepNum,
learnRate, lmbda)
else:
loop, stepNum, learnRate, lmbda = rf.Ask_Input_SGD(
loop, stepNum, learnRate, lmbda)
print(
"Hyper pameters: Loop({}), stepNum({}), learnRatio({}), lmbda({})\n"
.format(loop, stepNum, learnRate, lmbda))
# Start Training-
keepTraining = True
while (keepTraining):
start = time.time()
# Start Training-
net.Train(lstTrain,
loop,
stepNum,
learnRate,
lstV,
lmbda,
initialWeiBias,
trainOnlyDigit=digitIdOnly)
initialWeiBias = False
dT = time.time() - start
fnNetworkData1 = rf.Save_NetworkDataFile(net, fnNetworkData, loop,
stepNum, learnRate, lmbda,
dT, ".dnn")
keepTraining = ri.Ask_YesNo("Keep Training?", "y")
# Prediction ------------------------------------------------
if (not os.path.isfile(fnNetworkData1)):
fnNetworkData1= ".\\NetData\\{}_DNN_Digit.dnn". \
format(rn.RvNeuralNetwork.__name__)
# Ask DoPredict----------------------------------------------------
DoPredict = True
# if DoTraining: DoPredict = ri.Ask_YesNo("Predict digits?", "n")
# else: DoPredict = True
if DoPredict:
rn.Debug_Plot = True #ri.Ask_YesNo("Plot Digits?", "n")
# Predict_Digits(net, lstT)
rf.Predict_Digits_FromNetworkFile(fnNetworkData1, lstT, rn.Debug_Plot)
delta = d_cost(activations[-1],
np.array(output).reshape(len(output), 1)) * d_sigmoid(
zs[-1])
nudge_biases[-1] = delta
nudge_weights[-1] = np.dot(delta, activations[-2].transpose())
for i in range(2, self.num_layers):
delta = np.dot(self.weights[-i + 1].transpose(),
delta) * d_sigmoid(zs[-i])
nudge_biases[-i] = delta
nudge_weights[-i] = np.dot(delta, activations[-i - 1].transpose())
return nudge_biases, nudge_weights
random.seed(0)
def rand():
return random.gauss(0, 1)
training_data, validation_data, test_data = [
list(x) for x in mnist_loader.load_data_wrapper()
]
net = Network(784, 30, 10)
net.gradient_descent(training_data, 30, 10, 3.0, test_data)
print(2 + 2)
import mnist_loader
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
import network2
# net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
# net.large_weight_initializer()
# # net.SGD(list(training_data)[:1000], 400, 10, 0.5,
# # evaluation_data=test_data, lmbda = 0.1,
# # monitor_evaluation_cost=True, monitor_evaluation_accuracy=True,
# # monitor_training_cost=True, monitor_training_accuracy=True)
#
# # net.SGD(training_data, 30, 10, 0.5,
# # evaluation_data=test_data, lmbda = 5.0,
# # monitor_evaluation_accuracy=True, monitor_training_accuracy=True)
net = network2.Network([784, 100, 10], cost=network2.CrossEntropyCost)
net.large_weight_initializer()
net.SGD(training_data,
30,
10,
0.1,
lmbda=5.0,
evaluation_data=validation_data,
monitor_evaluation_accuracy=True)
# print("Sal!!!!!!!!!!!!")
#
# net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
# net.SGD(training_data, 30, 10, 0.1, lmbda = 5.0,
# evaluation_data=validation_data,
"""
Testing code for different neural network configurations.
Adapted for Python 3.5.2
Usage in shell:
python3.5 test.py
Network (network.py) parameters:
2nd param is epochs count
3rd param is batch size
4th param is learning rate (eta)
"""
# ----------------------
# - read the input data:
import mnist_loader
from expnd_dataset.expand_mnist import expand_mnist
expand_mnist()
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
filename="mnist_expanded.pkl.gz")
training_data = list(training_data)
# ---------------------
# - network.py example:
import network
save_model = "mnist_expanded.json"
net = network.Network([784, 30, 10])
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
net.save(save_model)
#net = network.load(save_model)
"""
梯度下降方法
:type training_data: tuples(x, y)
:param training_data: 输入，训练数据
:param epochs: 轮数，one epoch = numbers of iterations = N = 训练样本的数量/batch size
:param mini_batch_size: batch大小
:param eta: 学习率
:param test_data: 测试数据，若传入，则在每一轮(epoch)结束后用该测试数据评价一次网络
:return: 输出tuples(x, y)
"""
n_test = None
if test_data:
n_test = len(test_data)
n = len(training_data)
for j in range(1, epochs + 1):
random.shuffle(training_data)
mini_batches = [training_data[k:k + mini_batch_size] for k in range(0, n, mini_batch_size)]
for mini_batch in mini_batches:
self.update_mini_batch(mini_batch, eta)
if test_data:
print('Epoch {}: {} / {}'.format(j, self.evaluate(test_data), n_test))
else:
print('Epoch {} complete'.format(j))
if __name__ == '__main__':
from mnist_loader import load_data_wrapper
training_dat, validation_dat, test_dat = load_data_wrapper()
net = Network([784, 30, 10])
net.sgd(training_dat, 30, 10, 1.0, test_dat)
'''
Most of this code is based on the code of the original author:
Michał Dobrzański, 2016
[email protected]
Nevertheless, many changes and adjustments were made as described in the report.
'''
import mnist_loader
import network2
training_data, validation_data, test_data, input_dimension = mnist_loader.load_data_wrapper(
)
#this is done to save time in training
#mnist_loader.save(training_data, validation_data, test_data, input_dimension, 'PCA_data') #save dim_reduced data
#training_data, validation_data, test_data, input_dimension = mnist_loader.load('PCA_data') #load the data
training_data = list(training_data)
validation_data = list(validation_data)
input_layer = input_dimension
second_layer = 30
third_layer = 30
fourth_layer = 16
fifth_layer = 16
last_layer = 10
epochs = 5
batch_size = 1
eta = 0.1
lmbda = 5.0
n = len(training_data[:50000])
evaluation_data = validation_data[:10000] #could also be test_data
import matplotlib.cm as cm
# prvaite libraries---------------------------------------------
import mnist_loader
import RvNeuralNetworks as rn
from RvNeuralNetworks import *
import RvAskInput as ri
import RvMiscFunctions as rf
import RvActivationCost as ac
import PlotFunctions as pltFn
#%%
#使用 mnist_expanded.pkl.gz(250000筆） 準確率提高到 0.97
fn = ".datamnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
fnNetworkData1 = ".\\{}_NetData_DontDelete.txt".format(
rn.RvNeuralNetwork.__name__)
fnNetworkData2 = ".\\{}_NetData_DropOut.txt".format(
rn.RvNeuralNetwork.__name__)
fnNetworkData3 = ".\\{}_NetData_CnvLyr.txt".format(rn.RvNeuralNetwork.__name__)
#
accur1, t1 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData1, lstT, False)
accur2, t2 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData2, lstT, False)
accur3, t3 = pltFn.Predict_Digits_FromNetworkFile(fnNetworkData3, lstT)
print("FullConnected Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur1, t1))
print("DropOut Layer:\n Accu:{}, Time:{:.3} sec\n".format(accur2, t2))
# Make those columns into a array of 8-bits pixels
# This array will be of 1D with length 784
# The pixel intensity values are integers from 0 to 255
pixels = np.array(pixels, dtype='uint8')
# Reshape the array into 28 x 28 array (2-dimensional array)
pixels = pixels.reshape((28, 28))
# Plot
plt.title('Label is {label}'.format(label=label))
plt.imshow(pixels, cmap='gray')
plt.show()
# Using MNIST
train_data, val_data, test_data = mnist_loader.load_data_wrapper()
# ugly patch for changing the format in which data is stored in the array
trans = zip(*test_data[0:20])
samples = trans[0]
s_list = []
for i in range(len(samples)):
flatten = [val for sublist in samples[i] for val in sublist]
s_list.append(flatten)
samples = s_list
y_vec = trans[1]
# example use to show the numbers in class
# module: ai_f18_hw06.py
# Kody Sanchez
# A01514541
###############################
import numpy as np
import pickle as cPickle
from sklearn import svm, datasets, metrics
from sklearn.model_selection import train_test_split
import random
from mnist_loader import load_data_wrapper
## load MNIST
mnist_train_data, mnist_test_data, mnist_valid_data = \
load_data_wrapper()
# define numpy arrays for MNIST data; dc stands for
# data conversion.
mnist_train_data_dc = np.zeros((50000, 784))
mnist_test_data_dc = np.zeros((10000, 784))
mnist_valid_data_dc = np.zeros((10000, 784))
# define numpy arrays for MNIST targets
mnist_train_target_dc = None
mnist_test_target_dc = None
mnist_valid_target_dc = None
# here is how we reshape mnist data for sklearn decision trees.
def reshape_mnist_d(mnist_data, mnist_data_dc):
def Main():
#Load MNIST ****************************************************************
#Use mnist.pkl.gz(50000 data） Accuracy 0.96
#Use mnist_expanded.pkl.gz(250000 data） Accuracy 0.97
fn = "..\\data\\mnist.pkl.gz" #".datamnist_expanded.pkl.gz"
lstTrain, lstV, lstT = mnist_loader.load_data_wrapper(fn)
lstTrain = list(lstTrain)
lstV = list(lstV)
lstT = list(lstT)
# Load digit Images lstTrain[0].shape = (pxls, label) = (784, 1)
# imgPxls = lstTrain[0][0].shape[0]
# digitImages = rn.RvNeuralEnDeCoder.Load_DigitImages( ".\\Images\\Digits\\", imgPxls)
path = "..\\TmpLogs\\"
rfi.ForceDir(path)
fnNetworkData = "{}{}_EDC".format(path, rn.RvNeuralEnDeCoder.__name__)
#Hyper pameters -------------------------------------------
loop = 10 # loop effect，10, 30 all above 0.95
stepNum = 10 # stepNum effect, 10->0.9, 100->0.5
learnRate = 0.1 # learnRate and lmbda will affect each other
lmbda = 5.0 #add lmbda(Regularization) to solve overfitting
endecoder = None
sTrain = "y"
# Training ***********************************************************
# Ask DoTraining-
LoadAndTrain = ri.Ask_YesNo("Load exist model and continue training?", "n")
if LoadAndTrain:
digitIdOnly = -1
fns, shortFns = rfi.Get_FilesInFolder(".\\NetData\\", [".endecoder"])
aId = ri.Ask_SelectItem("Select EnDecoder file", shortFns, 0)
fn1 = fns[aId]
endecoder = rn.RvNeuralEnDeCoder(fn1)
initialWeights = False
sTrain = "n"
encoder, decoder = endecoder.Get_Encoder_Decoder()
else:
"""
[784,50,10], loop=100, 0.9725
"""
#buildDeNoiseModel = ri.Ask_YesNo("Build DeNoise Model?", "n")
# Create RvNeuralEnDeCoder----------------------------------------------
inputNeusNum = len(lstTrain[0][0])
#lyr2NeuNum = len(lstTrain[0][1])
# [784,256,128,10] is suggested ----------------------------
# [784, 256, 128, 10, 128, 256, 784 ] -> 0.9395 ... tested 10 epochs
# [784, 400, 20, 400, 784] -> 0.9526 ... tested 5 epochs
lyrsNeus = [inputNeusNum, 50] # 512, 256,128]
lyrsNeus = ri.Ask_Add_Array_Int(\
"Input new layer Neurons num.", lyrsNeus, 50)
bottleneckNeuNum = ri.Ask_Input_Integer(\
"Input BottleNeck(Code) Layer Neurons num.", 10)
lyrsNeus.append(bottleneckNeuNum)
for nNeu in reversed(lyrsNeus[1:-1]):
lyrsNeus.append(nNeu)
lyrsNeus.append(inputNeusNum)
digitIdOnly = -1
digitIdOnly = ri.Ask_Input_Integer(
"Build only Digit (0~9, -1 = All Digits): ", digitIdOnly)
#net = RvNeuralEnDeCoder(
# RvNeuralEnDeCoder.LayersNeurons_To_RvNeuralLayers(lyrsNeus))
#net = RvNeuralEnDeCoder.Class_Create_LayersNeurons(lyrsNeus)
endecoder = rn.RvNeuralEnDeCoder(lyrsNeus) # ([784,50,10])
initialWeights = True
# Training ***********************************************************
DoTrain = ri.Ask_YesNo("Do Training?", sTrain)
if DoTrain:
fnNetworkData = "{}_{}Lyr".format(fnNetworkData,
len(endecoder.NeuralLayers))
endecoder.DoPloatWeights = ri.Ask_YesNo("Plot Neurons Weights?", 'n')
# Ask nmtivation ------------------_----------
enumActivation = ri.Ask_Enum("Select Activation method.",
nm.EnumActivation,
nm.EnumActivation.afSigmoid)
for lyr in endecoder.NeuralLayers:
lyr.Set_EnumActivation(enumActivation)
endecoder.NetEnableDropOut = ri.Ask_YesNo("Execute DropOut?", "n")
if endecoder.NetEnableDropOut:
enumDropOut = ri.Ask_Enum("Select DropOut Method.",
nm.EnumDropOutMethod,
drpOut.eoSmallActivation)
rn.gDropOutRatio = ri.Ask_Input_Float("Input DropOut ratio.",
rn.gDropOutRatio)
endecoder.Set_DropOutMethod(enumDropOut, rn.gDropOutRatio)
# Auto-Caculate proper hyper pameters ---
DoEvaluate_ProperParams = ri.Ask_YesNo(
"Auto-Caculating proper hyper pameters?", "n")
if DoEvaluate_ProperParams:
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_lmbda(
endecoder, endecoder.Train, lstTrain[:1000], lstV[:500], loop,
stepNum, learnRate, lmbda)
loop, stepNum, learnRate, lmbda = rf.Evaluate_BestParam_learnRate(
endecoder, endecoder.Train, lstTrain[:1000], lstV[:500], loop,
stepNum, learnRate, lmbda)
else:
loop, stepNum, learnRate, lmbda = rf.Ask_Input_SGD(
loop, stepNum, learnRate, lmbda)
print(
"Hyper pameters: Loop({}), stepNum({}), learnRatio({}), lmbda({})\n"
.format(loop, stepNum, learnRate, lmbda))
# Start Training-
keepTraining = True
while (keepTraining):
start = time.time()
encoder, decoder = endecoder.Build_Encoder_Decoder( \
lstTrain, loop, stepNum, learnRate, lmbda, initialWeights, digitIdOnly)
initialWeights = False
dT = time.time() - start
rf.Save_NetworkDataFile(endecoder, fnNetworkData, loop, stepNum,
learnRate, lmbda, dT, ".endecoder")
fn1 = rf.Save_NetworkDataFile(encoder,
"{}_Encoder".format(fnNetworkData),
loop, stepNum, learnRate, lmbda, dT,
".encoder")
fn2 = rf.Save_NetworkDataFile(decoder,
"{}_Decoder".format(fnNetworkData),
loop, stepNum, learnRate, lmbda, dT,
".decoder")
keepTraining = ri.Ask_YesNo("Keep Training?", "y")
decoder = rn.RvNeuralEnDeCoder(fn2)
encoder = rn.RvNeuralEnDeCoder(fn1)
noiseStrength = ri.Ask_Input_Float("Input Noise Strength.", 0.7)
rf.Test_Encoder_Decoder(encoder, decoder, lstT, 10, "", noiseStrength)
def main():
config = configparser.ConfigParser()
config.read('config_mcsample.ini')
# select correct data
train_size = int(config['DATA']['train_size'])
test_size = int(config['DATA']['test_size'])
if config['DATA']['dataset'] == "simulated":
num_cov = int(config['DATA']['num_cov'])
mu = float(config['DATA']['mu'])
std = float(config['DATA']['std'])
range_cov = float(config['DATA']['range_cov'])
range_coef = float(config['DATA']['range_coef'])
range_bias = float(config['DATA']['range_bias'])
generator = generate_data.GenerateData(
num_cov, mu, std, range_cov, range_coef, range_bias,
seed=100) # Maybe add to config file..
X_train, y_train, _ = generator.generate(seed=15,
sample_size=train_size)
X_test, y_test, _ = generator.generate(seed=16, sample_size=test_size)
network_mcsample_plots(X_train, X_test, y_train, y_test,
config['RS NN PARAMS'], config['MCSAMPLE'])
if config['DATA']['dataset'] == "mnist":
train_full, validate_full, test_full = mnist_loader.load_data_wrapper(
) # we wont need validate dataset
X_train = np.array(train_full[0][:train_size])
y_train = np.array(train_full[1][:train_size])
X_test = np.array(test_full[0][:test_size])
y_test = np.array(test_full[1][:test_size])
network_mcsample_plots(X_train, X_test, y_train, y_test,
config['RS NN PARAMS'], config['MCSAMPLE'])
if config['DATA']['dataset'] == "iris":
from sklearn import datasets
from sklearn.model_selection import KFold
fold = True
data = datasets.load_iris()
y_train = pd.get_dummies(data.target).values
X_train = data.data
splits = int(config['DATA']['splits']) # Number of splits selected.
kf = KFold(splits)
kf.get_n_splits(X_train)
kf.split(X_train)
Xtr = []
Xtest = []
ytr = []
ytest = []
for train_index, test_index in kf.split(X_train):
# train
# print("TRAIN:", train_index, "TEST:", test_index)
Xtr.append(X_train[train_index])
ytr.append(y_train[train_index])
# test
Xtest.append(X_train[test_index])
ytest.append(y_train[test_index])
network_mcsample_plots(Xtr,
Xtest,
ytr,
ytest,
config['RS NN PARAMS'],
config['MCSAMPLE'],
fold=fold)
""" Python Created by Zhixuan Wang 01/29/2015 14:11 this script uses the method provided to train and predict """ # Libraries import mnist_loader as loader import network as nw data = loader.load_data_wrapper() training_data = data[0] validation_data = data[1] test_data = data[2] net = nw.Network([784, 100, 10]) net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
for i in xrange(length):
# print('i {}'.format(i))
x = raw_data[i][0]
y = raw_data[i][1]
x_final = []
y_final = []
x_len = len(np.atleast_1d(x))
y_len = len(np.atleast_1d(y))
for k in xrange(x_len):
x_k = x[k][0]
x_final.append(x_k)
for j in xrange(y_len):
y_j = y[j][0]
y_final.append(y_j)
finalData.append((x_final, y_final))
print('Formatting ended')
return finalData
if __name__ == '__main__':
print('Loading starts')
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
print('Loading ended')
network = MultiLayerNetwork([784, 30, 10])
net = Network([784,30,10])
network.SGD(training_data, 30, 10, 3.0, test_data=test_data)
def test():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net = Network([784,30,10])
net.SGD(training_data, 30, 10, 3.0)
