from mlp import *
def loadDataSet(filepath):
f=gzip.open(filepath,'rb')
#self.train_set,self.valid_set,self.test_set=cPickle.load(f)
data=cPickle.load(f)
f.close()
return data
'''
mlp has lots of things to consider. you need to use the momentum factor or inertial factor to search the optimal result
you need to think how many samples in a batch, you can use small number of them, even one sample in a batch, but you can't use too much in a batch, Batch means you add the errors from all samples and upate the weigth once according to the total error. you use small number batch only means you will converge very slow or get a loca optimum. but it won't influence too much. but you use large number batch, you will crash. because every time you update the delta_weigth too big. it just like you use a large step. the large step length. training or learning process in fact is a optimization process. if you can find a very easy way to search the parameter of a function. you don't need neural network at all. you just use math to solve it.
'''
if __name__=="__main__":
mlp=mlp()
mlp.hidden_neurons=20
mlp.output_neurons=10
#this is the most important parameter, you have no need to care about the error_goal
mlp.max_iteration=1000
mlp.error_goal=0.01
mlp.batch=10
data=loadDataSet('../data/mnist.pkl.gz')
#because the sum of errors in every batch as one update, if you use 10000 samples in a batch, the sum error too large, and it will casue the activation function full,cause the output keep the same time, because the output neurons are full. so the output value is -1 or 1.so y(1-y)=0 or -2, so the gradient is 0 or -2 always, finally the gradient is 0 and cause can't update the weigth
train_samples=10000
x=data[0][0][0:train_samples]
y=data[0][1][0:train_samples]
#sigmoid needs the [-1,1] range, not the [0,1]
from sys import *
from mlp import *
if __name__ == "__main__":
if len(argv) < 2:
print "add the argument"
exit()
data_part = argv[1]
data = load_data(data_part)
train = data[0]
valid = data[1]
test = load_test(data_part)
print "Training phase"
m_p = mlp(train[0],train[1],valid[0],valid[1],\
h1=40,l_rate=0.01,init_var=0.1,momentum_rate=0.5)
print "Testing phase"
success_rate = test_mlp(test, m_p)
print " -> The success rate on the test set is", success_rate
from sys import *
from mlp import *
if __name__ == "__main__":
if len(argv) < 2:
print "add the argument"
exit()
data_part = argv[1]
data = load_data(data_part)
train = data[0]
valid = data[1]
test = load_test(data_part)
print "Training phase"
m_p = mlp(train[0],train[1],valid[0],valid[1],\
h1=40,l_rate=0.01,init_var=0.1,momentum_rate=0.5)
print "Testing phase"
success_rate = test_mlp(test,m_p)
print " -> The success rate on the test set is", success_rate
def climbThatHill():
t = time.time()
j = 10
# Check out mlp's printParameter() for explination of this list.
# I should plot this as a function of time.
bestParamList = [1, 1, 20, 4, 0.01, 0.01, 10, 256, 512, 1024, 2048]
bestNet = mlp(trainingData, trainingTargets, bestParamList, j)
bestNet.printParameters()
bestError = bestNet.earlystopping(trainingData, trainingTargets, validationData, validationTargets)
dt = 0.0
iterations = 0
moves = 0
bestErrors = [bestError]
p,m1,m2 = bestNet.confusion(testData,testTargets,True)
confusion0 = [p]
confusion1 = [m1]
confusion2 = [m2]
times = [dt]
p,m1,m2
paramList = bestParamList[:]
while dt < 300:
iterations += 1
np.random.seed()
for i in range(len(paramList)):
# this tends to multiply or divide numbers by ~e
# so it is a scale invariant smooth unimodal distribution
paramList[i] = bestParamList[i] * math.exp(random.normalvariate(0, 0.3))
currNet = mlp(trainingData, trainingTargets, paramList, j)
# currNet.printParameters()
# paramList
# currNet.setParameters(paramList)
currError = currNet.earlystopping(trainingData, trainingTargets, validationData, validationTargets)
dt = (time.time() - t)
if currError < bestError:
print "***************** MOVE ACCEPTED"
moves += 1
bestParamList = paramList[:]
bestNet = currNet
bestNet.printParameters()
bestError = currError
print "bestError so far: ", bestError
bestErrors.append(bestError)
p,m1,m2 = bestNet.confusion(testData,testTargets,False)
confusion0.append(p)
confusion1.append(m1)
confusion2.append(m2)
times.append(dt)
bestErrors.append(bestError)
confusion0.append(p)
confusion1.append(m1)
confusion2.append(m2)
times.append(dt)
# plt.figure(1)
# plt.plot(times, bestErrors)
# plt.savefig("bestErrors.png")
# plt.figure(2)
# plt.plot(times, confusion0)
# plt.savefig("P_correct.png")
# plt.figure(3)
# plt.plot(times, confusion1)
# plt.savefig("worst_P_correct_|_yMax.png")
# plt.figure(4)
# plt.plot(times, confusion2)
# plt.savefig("worst_P_correct_|_target.png")
bestNet.confusion(testData,testTargets,True)
print "Iterations: ", iterations
if (iterations > 0):
print "P(Move Accept): ", moves/float(iterations)
else:
print "P(Move Accept): -1"
print "Best Error: ", bestError
word_vec_model = "new"
relations_file = ""
train_file = "../data/kbp_train.txt"
test_file = "../data/kbp_dev.txt"
num_words = 2675308
word_dim = 300
hidden_dimension = 1000
output_dimension = 42
activ = 'sigmoid'
nepochs = 10
batch_size = 1000
printevery = 10000
saveevery = 2000000
lrate = 0.0001
ldecay = 0.000001
reg = 0.01
output_file = "../data/test/test_out/predictions_ep_" + str(nepochs) + "_lr_" + str(lrate) + "_reg_" + str(reg) + "_bs_" + str(batch_size) + "_lrdecay_" + str(ldecay) + ".mlpout"
model_loc = "saved_models/hd_" + str(hidden_dimension) + "_lr_" + str(lrate) + "_reg_" + str(reg) + "_bs_" + str(batch_size) + "_lrdecay_" + str(ldecay)
wv = load_word_vectors(word_vec_file, word_dim)
#rel = read_relations(relations_file)
rel = None
print('Done reading relations and word vectors.')
mod = mlp(wv, activ, num_words, word_dim, hidden_dimension, output_dimension, reg, lrate, ldecay, batch_size, nepochs)
mod.train(train_file, model_loc, printevery, saveevery)
mod.test(test_file, output_file)
import matplotlib.pyplot as plt
if __name__ == "__main__":
accuracy=[]
memory_used=[]
time=[]
algos=["SVM","MLP","DTree"]
fuzzy()
accuracy.append(svm_algo())
time.append(timeit.repeat(repeat=1))
memory_used.append(memory_usage_psutil())
accuracy.append(mlp())
time.append(timeit.repeat(repeat=1))
memory_used.append(memory_usage_psutil())
accuracy.append(dtree())
time.append(timeit.repeat(repeat=1))
memory_used.append(memory_usage_psutil())
fig=plt.figure()
plt.title("Execution Time Vs Algorithm")
plt.plot(algos,time)
plt.xlabel('Algorithm')
plt.ylabel('Execution Time in sec')
plt.show(fig)
fig1 = plt.figure()
plt.title("Memory Vs Algorithm")
test_file = "../data/kbp_dev.txt"
num_words = 2675308
word_dim = 300
hidden_dimension = 1000
output_dimension = 42
activ = 'sigmoid'
nepochs = 10
batch_size = 1000
printevery = 10000
saveevery = 2000000
lrate = 0.0001
ldecay = 0.000001
reg = 0.01
output_file = "../data/test/test_out/predictions_ep_" + str(
nepochs) + "_lr_" + str(lrate) + "_reg_" + str(reg) + "_bs_" + str(
batch_size) + "_lrdecay_" + str(ldecay) + ".mlpout"
model_loc = "saved_models/hd_" + str(hidden_dimension) + "_lr_" + str(
lrate) + "_reg_" + str(reg) + "_bs_" + str(batch_size) + "_lrdecay_" + str(
ldecay)
wv = load_word_vectors(word_vec_file, word_dim)
#rel = read_relations(relations_file)
rel = None
print('Done reading relations and word vectors.')
mod = mlp(wv, activ, num_words, word_dim, hidden_dimension, output_dimension,
reg, lrate, ldecay, batch_size, nepochs)
mod.train(train_file, model_loc, printevery, saveevery)
mod.test(test_file, output_file)
