def __init__(self,
numpy_rng=None,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500, 500],
n_outs=10):
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30))
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
rbm_layer = RBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.error(self.y)
def infer(w):
test = pd.read_csv('../data/census-income.test.clean',
header=None,
delim_whitespace=True)
test_y = test.values[:, -1]
X = test.values[:, :-1]
X = np.array(X, dtype='float64')
test_X = np.copy(X)
normal_list = [0, 126, 210, 211, 212, 353, 499]
for i in normal_list:
test_X[:, i] = (X[:, i] - X[:, i].mean()) / (X[:, i].std())
lr = LogisticRegression()
print(lr.score(test_X, test_y, w))
lr.F1(test_X, test_y, w)
def compare_lr():
# Instantiate the Logistic Regression model
model = LogisticRegression(l2penalty=0, verbose=True)
# Evaluate and plot the model with different learning rates
for exp in (-2, -4, -6):
accuracies_tr = []
accuracies_ts = []
iterations = [i for i in range(1, ITERATIONS + 1)]
# Fit the model. Note that while fitting we are predicting due to the presence of a callback.
model.fit(x_tr,
y_tr,
lr=10**exp,
maxit=ITERATIONS,
callback=[accuracy_callback, accuracies_tr, accuracies_ts])
# Plot the accuracies
fig = plt.figure()
plt.plot(iterations,
accuracies_tr.copy(),
label='Train',
color='blue',
linewidth=1)
plt.plot(iterations,
accuracies_ts.copy(),
label='Test',
color='green',
linewidth=1)
plt.title('Train vs Test with lr = 10^%s' % exp)
plt.ylabel('Accuracy')
plt.xlabel('Iteration')
plt.grid(True, linestyle=':')
plt.legend(loc="upper right")
plt.show()
fig.savefig('logistic_accuracy_lr10^%s.png' % exp)
def main():
if args.modelIdx == '1':
model = LogisticRegression()
elif args.modelIdx == '2':
model = SupportVectorMachine()
elif args.modelIdx == '3':
tssp = [0.01, 0.03, 0.05, 0.08, 0.1, 0.15]
analysis_1(tssp)
return
elif args.modelIdx == '4':
tssp = [0.01, 0.03, 0.05, 0.08, 0.1, 0.15]
analysis_2(tssp)
return
else:
return
model.train_from_csv(args.trainingDataFilename)
model.test_from_csv(args.testDataFilename)
def calculator(self):
print('enter calculator')
self.X, self.y = self.read_data()
self.lr = LogisticRegression(n_iter=70, eta=0.005, batch_size=10, gammar=0.5)
while not self._begin_cal:
time.sleep(1)
start_time = time.time()
st_clk = time.clock()
print('start calculator')
param_list = self.GetParams()
param_list, loss = self.algorithm(param_list, 0)
# index_epoch = (self._data_end - self._data_begin) // self.lr.batch_size
index = 0
while index < self._max_epoch:
index += 1
self.SetParams(param_list)
param_list = self.GetParams()
param_list, loss = self.algorithm(param_list, index)
if index % 2000 == 0:
print("Calculator.calculator::", index)
print(loss)
print('stop calculator', time.time() - start_time)
print('CPU time', time.clock() - st_clk)
def cross_validate(csv_file_name,
tssp,
losses_file_name,
new_feature=False,
debug=False):
'''
Perform 10-fold incremental cross validation.
'''
total_num = 2000
num_words = 4000
lists_of_dict = []
losses = zeros((3, len(tssp), 10)) # #models, #tss, #folds
sklosses = zeros((2, len(tssp), 10))
generate_train_and_test_files_cv(csv_file_name, 10)
for i in range(10):
lists_of_dict.append(csv_to_dict('cv%d.dat' % (i)))
for i, proportion in enumerate(tssp):
for j in range(10):
# Contruct train set
training_lists_of_dict = lists_of_dict[:j] + lists_of_dict[j + 1:]
training_list_of_dict = [
item for sublist in training_lists_of_dict for item in sublist
]
testing_list_of_dict = lists_of_dict[j]
# Randomly select samples
random_indices = permutation(len(training_list_of_dict))
random_indices = random_indices[:int(total_num * proportion)]
training_list_of_dict = [
training_list_of_dict[k] for k in random_indices
]
# Find the word features
feature_words = construct_word_feature(training_list_of_dict,
num_words)
# Extract features and labels
training_X, training_y = extract_word_feature_and_label(
training_list_of_dict, feature_words, new_feature=new_feature)
testing_X, testing_y = extract_word_feature_and_label(
testing_list_of_dict, feature_words, new_feature=new_feature)
# NBC
M = nbc_train(feature_words,
training_X,
training_y,
new_feature=new_feature)
losses[0, i, j], _ = nbc_test(M, testing_X, testing_y)
# LR
lr = LogisticRegression()
lr.train(training_X, training_y)
losses[1, i, j] = lr.test(testing_X, testing_y)
# SVM
svm = SupportVectorMachine()
svm.train(training_X, training_y)
losses[2, i, j] = svm.test(testing_X, testing_y)
# Libary functions
if debug:
training_y[training_y == 0] = -1
testing_y[testing_y == 0] = -1
sklr = skLogisticRegresssion()
sklr.fit(training_X.T, training_y)
sklosses[0, i, j] = 1 - sklr.score(testing_X.T, testing_y)
sksvm = skSupportVectorMachine()
sksvm.fit(training_X.T, training_y)
sklosses[1, i, j] = 1 - sksvm.score(testing_X.T, testing_y)
save(losses_file_name, losses)
save('debug_' + losses_file_name, sklosses)
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 25 18:45:40 2016
@author: NISHIT
"""
import time
start_exec_time = time.time()
from sentiment_reader import SentimentCorpus
from lr import LogisticRegression
if __name__ == '__main__':
dataset = SentimentCorpus()
lr = LogisticRegression()
lr.LR(dataset.train_X, dataset.train_y,dataset.test_X, dataset.test_y)
print("Execution time: --- %s seconds ---" % (time.time() - start_exec_time))
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=784,
hidden_layer_sizes=[500, 500],
n_out=10,
corruption_levels=[0.1, 0.1]):
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layer_sizes) #网络层数
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
self.x = T.matrix("x") #图像序列
self.y = T.ivector("y") #标签
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
else:
input_size = hidden_layer_sizes[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
"""HIddenLayer
def __init__(self,rng,input,n_in,n_out,
W=None,b=None,activation=T.tanh):
"""
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layer_sizes[i],
activation=T.nnet.sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
#创建一个dA与HiddenLayer共享权值
"""dA
def __init__(self,numpy_rng,theano_rng=None,input=None,
n_visible=784,n_hidden=500,
W=None,bhid=None,bvis=None):
"""
dA_layer = dA(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i - 1],
W=sigmoid_layer.W,
bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
"""LogisticRegression
def __init__(self,input,n_in,n_out):
"""
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layer_sizes[-1],
n_out=n_out)
self.params.extend(self.logLayer.params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.error(self.y)
class sdA(object):
"""
n_ins:输入层的维度（第一层为数据输入层）
hidden_layer_sizes:中间层大小
n_out:网络的输出大小
corruption_levels:每层的corruption程度
"""
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=784,
hidden_layer_sizes=[500, 500],
n_out=10,
corruption_levels=[0.1, 0.1]):
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layer_sizes) #网络层数
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
self.x = T.matrix("x") #图像序列
self.y = T.ivector("y") #标签
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
else:
input_size = hidden_layer_sizes[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
"""HIddenLayer
def __init__(self,rng,input,n_in,n_out,
W=None,b=None,activation=T.tanh):
"""
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layer_sizes[i],
activation=T.nnet.sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
#创建一个dA与HiddenLayer共享权值
"""dA
def __init__(self,numpy_rng,theano_rng=None,input=None,
n_visible=784,n_hidden=500,
W=None,bhid=None,bvis=None):
"""
dA_layer = dA(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i - 1],
W=sigmoid_layer.W,
bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
"""LogisticRegression
def __init__(self,input,n_in,n_out):
"""
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layer_sizes[-1],
n_out=n_out)
self.params.extend(self.logLayer.params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of training,
# defined as the negative log likelihood
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.error(self.y)
"""
下面这个函数是为!!!每i层!!!生成一个Theano function类型的预训练函数，
返回值是一个list，list中就是每层的预训练函
为了能在训练中更改 the corruption level or the learning rate，
我们定义Theano variables。
"""
def pretraining_functions(self, train_set_x, batch_size):
index = T.lscalar('index')
corruption_level = T.scalar('corruption')
learning_rate = T.scalar('lr')
batch_begin = index * batch_size
batch_end = batch_begin + batch_size
pretrain_fns = []
for dA in self.dA_layers:
cost, updates = dA.get_cost_update(corruption_level, learning_rate)
fn = theano.function(
inputs=[
index,
theano.In(corruption_level, value=0.2),
theano.In(learning_rate, value=0.1)
],
outputs=cost,
updates=updates,
givens={self.x: train_set_x[batch_begin:batch_end]})
pretrain_fns.append(fn)
return pretrain_fns
"""
创建微调函数
Once all layers are pre-trained,
the network goes through a second stage of training called fine-tuning.
降噪自编码完成之后，网络进行监督学习得到误差再微调参数
"""
def build_finetune_functions(self, datasets, batch_size, learning_rate):
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
n_valid_batches = valid_set_x.get_value(
borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(
borrow=True).shape[0] // batch_size
index = T.lscalar('index')
gparams = T.grad(self.finetune_cost, self.params)
updates = [(param, param - gparam * learning_rate)
for param, gparam in zip(self.params, gparams)]
train_fn = theano.function(
inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x:
train_set_x[index * batch_size:(index + 1) * batch_size],
self.y:
train_set_y[index * batch_size:(index + 1) * batch_size]
},
name='train')
test_score_i = theano.function(
inputs=[index],
outputs=self.errors,
givens={
self.x:
test_set_x[index * batch_size:(index + 1) * batch_size],
self.y: test_set_y[index * batch_size:(index + 1) * batch_size]
},
name='test')
valid_score_i = theano.function(
inputs=[index],
outputs=self.errors,
givens={
self.x:
valid_set_x[index * batch_size:(index + 1) * batch_size],
self.y:
valid_set_y[index * batch_size:(index + 1) * batch_size]
},
name='valid')
def valid_score():
return [valid_score_i(i) for i in range(n_valid_batches)]
def test_score():
return [test_score_i(i) for i in range(n_test_batches)]
return train_fn, valid_score, test_score
import sys
from lr import LogisticRegression
#from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
X, Y = [], []
for line in sys.stdin:
p = line.strip().split(' ')
X.append([float(x) for x in p[1:-1]])
Y.append(int(p[-1]))
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)
W = [0] * len(X[0])
m = LogisticRegression()
W = m.fit(X_train, Y_train, W, 1.0, 50000)
print '== test on kflearn:', m.score(X_test, Y_test, W)
print '== train on kflearn:', m.score(X_train, Y_train, W)
print '== all on kflearn:', m.score(X, Y, W)
'''
m = LogisticRegression()
m.fit(X_train, Y_train)
print '== test on sklearn:', m.score(X_test,Y_test)
'''
def main():
# load training data
print("####################################################")
print("Data preprocessing...")
filename = 'data/train.csv'
Dict = transform(filename)
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
Dict['data'],
Dict['target'],
test_size=test_size,
random_state=random_state)
train = pd.concat([X_train, y_train], axis=1)
print("Cross validate on 10% of the whole dataset, training data shape: ",
train.shape)
thresh = nan_thresh * train.shape[1]
train = train.dropna(thresh=thresh)
#train = train.dropna(subset = ['Income','HouseholdStatus','EducationLevel'])
train = fill_missing(train, strategy, isClassified)
y = train['Happy']
df = train.drop('Happy', 1)
numeric_col = df[['YOB', 'votes']]
#numeric_col = fill_missing(numeric_col,'mean',False)
normalized = pd.DataFrame(normalize(numeric_col))
categorical_col = df[[
'Income', 'HouseholdStatus', 'EducationLevel', 'Party'
]]
#categorical_col = fill_missing(categorical_col,'median',False)
binary_col = df.drop(['UserID','YOB','Income','HouseholdStatus',\
'EducationLevel','Party','votes'],axis = 1)
encoder = OneHotEncoder()
#one_hot = pd.get_dummies(categorical_col)
one_hot = pd.DataFrame(encoder.fit_transform(categorical_col).toarray())
#X = pd.concat([normalized,one_hot,binary_col],axis = 1)
X = np.hstack((normalized, one_hot, binary_col))
#X = np.hstack((one_hot,binary_col))
X_test = fill_missing(X_test, strategy, isClassified)
UserID = X_test['UserID']
categorical_col = X_test[[
'Income', 'HouseholdStatus', 'EducationLevel', 'Party'
]]
numeric_col = X_test[['YOB', 'votes']]
binary_col = X_test.drop(['UserID','YOB','Income','HouseholdStatus',\
'EducationLevel','Party','votes'],axis = 1)
one_hot = encoder.fit_transform(categorical_col).toarray()
normalized = normalize(numeric_col)
#X = np.hstack((one_hot,binary_col))
X_test = np.hstack((normalized, one_hot, binary_col))
X_a = X
X_b = X_test
y_a = y
y_b = y_test
print("Training data shape (After drop NaN and one hot encoding): ",
X_a.shape)
print("Cross validation data shape (after one hot encoding): ", X_b.shape)
#X_a, X_b, y_a, y_b = cross_validation.train_test_split(X, y, test_size=0.1, random_state=0)
print("####################################################")
### use the logistic regression
print('Train the logistic regression classifier...')
start = time.time()
clf_lr = LogisticRegression()
clf_lr = clf_lr.fit(X_a, y_a)
clf_lr_predict = clf_lr.predict(X_b)
end = time.time()
print("Accuracy of logistic regression is ",
(y_b == clf_lr_predict).sum() / y_b.shape[0])
print("Time of of logistic regression is %.4fs." % (end - start))
start = time.time()
lr_model = linear_model.LogisticRegression()
lr_model = lr_model.fit(X_a, y_a)
lr_model_predict = lr_model.predict(X_b)
print("Scikit learn n_iter_: ", lr_model.n_iter_)
end = time.time()
print("Accuracy of scikit-learn logistic Regression is ",
(y_b == lr_model_predict).sum() / y_b.shape[0])
print("Time of of scikit-learn logistic regression is %.4fs." %
(end - start))
print(" ")
### use the naive bayes
print('Train the naive bayes classifier...')
start = time.time()
clf_nb = NaiveBayes()
clf_nb = clf_nb.fit(X_a, y_a)
clf_nb_predict = clf_nb.predict(X_b)
end = time.time()
print("Accuracy of naive bayes is ",
(y_b == clf_nb_predict).sum() / y_b.shape[0])
print("Time of of naive bayes is %.4fs." % (end - start))
start = time.time()
nb_model = BernoulliNB()
nb_model = nb_model.fit(X_a, y_a)
nb_model_predict = nb_model.predict(X_b)
end = time.time()
print("Accuracy of scikit-learn Bernoulli NB is ",
(y_b == nb_model_predict).sum() / y_b.shape[0])
print("Time of of scikit-learn Bernoulli NB is %.4fs." % (end - start))
print(" ")
## use the svm
print('Train the SVM classifier...')
start = time.time()
svm_model = SVC(kernel=kernel)
svm_model = svm_model.fit(X_a, y_a)
svm_model_predict = svm_model.predict(X_b)
end = time.time()
print("Accuracy of scikit-learn SVM is ",
(y_b == svm_model_predict).sum() / y_b.shape[0])
print("Time of of scikit-learn SVM is %.4fs." % (end - start))
print(" ")
## use the random forest
print('Train the random forest classifier...')
start = time.time()
rf_model = RandomForestClassifier(n_estimators=n_estimators,
random_state=random_state)
rf_model = rf_model.fit(X_a, y_a)
rf_model_predict = rf_model.predict(X_b)
end = time.time()
print("Accuracy of scikit-learn RF is ",
(y_b == rf_model_predict).sum() / y_b.shape[0])
print("Time of of scikit-learn RF is %.4fs." % (end - start))
print(" ")
print("####################################################")
print("Start predicting test dataset...")
filename_test = './data/test.csv'
Dict = transform(filename_test)
X_test_data = Dict['data']
test = fill_missing(X_test_data, strategy, isClassified)
UserID = test['UserID'].astype(int)
categorical_col = test[[
'Income', 'HouseholdStatus', 'EducationLevel', 'Party'
]]
numeric_col = test[['YOB', 'votes']]
binary_col = test.drop(['UserID','YOB','Income','HouseholdStatus',\
'EducationLevel','Party','votes'],axis = 1)
one_hot = encoder.fit_transform(categorical_col).toarray()
normalized = normalize(numeric_col)
#X_test = np.hstack((one_hot,binary_col))
X_test = np.hstack((normalized, one_hot, binary_col))
print("Test data shape (after one hot encoding): ", X_test.shape)
# clf_lr predictions
lr_pred = clf_lr.predict(X_test)
lr_Happy = pd.DataFrame({'Happy': lr_pred})
predsLR = pd.concat([UserID, lr_Happy], axis=1)
predsLR.to_csv('./predictions/lr_predictions.csv', index=False)
# clf_nb predictions
nb_pred = clf_nb.predict(X_test)
nb_Happy = pd.DataFrame({'Happy': nb_pred})
predsLR = pd.concat([UserID, nb_Happy], axis=1)
predsLR.to_csv('./predictions/nb_predictions.csv', index=False)
# rf predictions
rf_pred = rf_model.predict(X_test)
rf_Happy = pd.DataFrame({'Happy': rf_pred})
predsLR = pd.concat([UserID, rf_Happy], axis=1)
predsLR.to_csv('./predictions/rf_predictions.csv', index=False)
# svm predictions
svm_pred = svm_model.predict(X_test)
svm_Happy = pd.DataFrame({'Happy': svm_pred})
predsLR = pd.concat([UserID, svm_Happy], axis=1)
predsLR.to_csv('./predictions/svm_predictions.csv', index=False)
print("Prediction saved.")
class DBN(object):
def __init__(self,
numpy_rng=None,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500, 500],
n_outs=10):
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30))
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
rbm_layer = RBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.error(self.y)
def pretraining_functions(self, train_set_x, batch_size, k):
index = T.lscalar('index')
learning_rate = T.scalar('lr')
batch_begin = index * batch_size
batch_end = batch_begin + batch_size
pretrain_fns = []
for rbm in self.rbm_layers:
cost, updates = rbm.get_cost_updates(learning_rate,
persistent=None,
k=k)
fn = theano.function(
inputs=[index, theano.In(learning_rate, value=0.1)],
outputs=cost,
updates=updates,
givens={self.x: train_set_x[batch_begin:batch_end]})
pretrain_fns.append(fn)
return pretrain_fns
def build_finetune_functions(self, datasets, batch_size, learning_rate):
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_valid_batches //= batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_test_batches //= batch_size
index = T.lscalar('index')
gparams = T.grad(self.finetune_cost, self.params)
updates = []
for param, gparam in zip(self.params, gparams):
updates.append((param, param - gparam * learning_rate))
train_fn = theano.function(
inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x:
train_set_x[index * batch_size:(index + 1) * batch_size],
self.y:
train_set_y[index * batch_size:(index + 1) * batch_size]
})
test_score_i = theano.function(
[index],
self.errors,
givens={
self.x:
test_set_x[index * batch_size:(index + 1) * batch_size],
self.y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
valid_score_i = theano.function(
[index],
self.errors,
givens={
self.x:
valid_set_x[index * batch_size:(index + 1) * batch_size],
self.y:
valid_set_y[index * batch_size:(index + 1) * batch_size]
})
def valid_score():
return [valid_score_i(i) for i in range(n_valid_batches)]
def test_score():
return [test_score_i(i) for i in range(n_test_batches)]
return train_fn, valid_score, test_score
def evaluate_lenet5(learning_rate=0.1,
n_epochs=200,
dataset='mnist.pkl.gz',
nkerns=[20, 50],
batch_size=500):
rng = numpy.random.RandomState(23455)
datasets = load_dataset(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
index = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
print '...... building the model ......'
layer0_input = x.reshape((batch_size, 1, 28, 28))
layer0 = LeNetConvPoolLayer(rng=rng,
input=layer0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2))
layer1 = LeNetConvPoolLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], 12, 12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2))
layer2_input = layer1.output.flatten(2)
layer2 = HiddenLayer(rng=rng,
input=layer2_input,
n_in=nkerns[1] * 4 * 4,
n_out=500,
activation=T.tanh)
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
cost = layer3.negative_log_likelihood(y)
test_model = theano.function(
[index],
layer3.error(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer3.error(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
params = layer3.params + layer2.params + layer1.params + layer0.params
grads = T.grad(cost, params)
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
print '...... training ......'
patience = 10000
patience_increase = 2
improvement_threshold = 0.995
validation_frequency = min(n_train_batches, patience // 2)
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print('training @ iter = ', iter)
cost_ij = train_model(minibatch_index)
if (iter + 1) % validation_frequency == 0:
validation_losses = [
validate_model(i) for i in range(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
if this_validation_loss < best_validation_loss:
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
test_losses = [
test_model(i) for i in range(n_test_batches)
]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print('Optimization complete.')
print(
'Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.1fs' % ((end_time - start_time)))
best_label = None
for beta in initial_beta:
tmp_rate = 0.
for i, test_X in enumerate(k_X_set):
test_y = k_y_set[i]
#generate train_X
train_X = []
train_y = []
for j in xrange(len(k_X_set)):
if j == i:
continue
train_X.append(k_X_set[j])
train_y.append(k_y_set[j])
train_X = np.vstack(train_X)
train_y = np.hstack(train_y)
lr = LogisticRegression(beta=beta)
lr.fit(train_X, train_y)
exp_y = lr.predict(test_X)
tmp_rate += 1.0 - (exp_y ^ test_y).sum() / float(len(test_y))
tmp_rate /= cv_k
if tmp_rate > best_rate:
best_initial_beta = beta
best_rate = tmp_rate
print best_initial_beta
print best_rate
lr = LogisticRegression(beta=best_initial_beta)
X = []
y = []
for i in xrange(cv_k - 1):
# -*- coding: utf-8 -*-
"""
LogisticRegression1 test runner
Chapter "Logistic Regression"
"""
# imports
import numpy as np
from lr import LogisticRegression
# load data
data = np.genfromtxt('iris2.tsv', dtype=[('X', float, 4), ('y', int)])
# learn model
clr = LogisticRegression()
clr.fit(data['X'], data['y'])
# test model
predict_y = clr.predict(data['X'])
# print results
for i in range(0, 100, 10):
print(i, data['y'][i], predict_y[i])
# print accuracy
print("Accuracy =", end=' ')
print(np.sum(data['y'] == predict_y, dtype=float) / predict_y.shape[0])
# print parameters
print("coef =", clr.coef_)
def compare_penalty():
penalties = (-8, -6, -4, -2, 0, 2)
# Instantiate the Logistic Regression model with no regularization
model_unreg = LogisticRegression(verbose=True)
# Fit the model.
model_unreg.fit(x_tr, y_tr, lr=10**(-4), maxit=ITERATIONS, tolerance=None)
# Predict results
pred_tr_unreg = model_unreg.predict(x_tr)
pred_ts_unreg = model_unreg.predict(x_ts)
# Evaluate
train_scores = BinaryScorer(y_tr,
pred_tr_unreg,
description='Training',
positive_class=1,
negative_class=0)
test_scores = BinaryScorer(y_ts,
pred_ts_unreg,
description='Testing',
positive_class=1,
negative_class=0)
accuracy_tr_unreg = train_scores.accuracy()
accuracy_ts_unreg = test_scores.accuracy()
accuracies_tr_reg = []
accuracies_ts_reg = []
# Evaluate and plot the model with different learning rates
for alpha in penalties:
# Instantiate the Logistic Regression model with regularization
model_reg = LogisticRegression(l2penalty=2**alpha, verbose=True)
# Fit the model.
model_reg.fit(x_tr,
y_tr,
lr=10**(-4),
maxit=ITERATIONS,
tolerance=None)
# Predict results
pred_tr_reg = model_reg.predict(x_tr)
pred_ts_reg = model_reg.predict(x_ts)
# Evaluate
train_scores = BinaryScorer(y_tr,
pred_tr_reg,
description='Training',
positive_class=1,
negative_class=0)
test_scores = BinaryScorer(y_ts,
pred_ts_reg,
description='Testing',
positive_class=1,
negative_class=0)
accuracies_tr_reg.append(train_scores.accuracy())
accuracies_ts_reg.append(test_scores.accuracy())
fig = plt.figure()
plt.plot(penalties,
accuracies_tr_reg,
'o-',
label='Train w/ reg',
color='blue')
plt.plot(penalties,
accuracies_ts_reg,
'o-',
label='Test w/ reg',
color='purple')
plt.plot([penalties[0], penalties[len(penalties) - 1]],
[accuracy_tr_unreg, accuracy_tr_unreg],
'-',
label='Train w/o reg',
color='red')
plt.plot([penalties[0], penalties[len(penalties) - 1]],
[accuracy_ts_unreg, accuracy_ts_unreg],
'-',
label='Test w/o reg',
color='orange')
plt.title('Regularization comparison, lr = 10^-4')
plt.ylabel('Accuracy')
plt.xlabel('Values of k, with alpha = 2^k')
plt.grid(True, linestyle=':')
plt.legend(loc="upper right")
plt.show()
fig.savefig('logistic_penalty_comp.png', dpi=300)
import numpy as np
from util import read_matlab, trans_label_to_onehot, calc_precision
from plot import show_img, show_100img
from lr import LogisticRegression, load_lr
from NeuralNetwork import NeuralNetwork
if __name__ == '__main__':
# Read data from mat file.
data = read_matlab('./ex3/ex3data1.mat')
X = data['X']
label = data['y']
# Trans the label to onehot
label_onehot = trans_label_to_onehot(label).transpose()
# Show img
show_100img(X)
lr = LogisticRegression(400, 10)
# This method uses the lib function, if you want to use self create train method, use function LogisticRegression.train
lr.lib_train(X, label_onehot, 1)
# You can use save and load function to save model file
# lr.save_model('./model/lr.model')
# lr = load_lr('./model/lr.model')
ans = lr.predict_class(X.transpose())
print(
'\033[1;32mThe accuracy of Logistsic regression is {}%.\033[0m'.format(
calc_precision(label, ans)))
# Neural network
nn_weight = read_matlab('./ex3/ex3weights.mat')
theta1 = nn_weight['Theta1']
theta2 = nn_weight['Theta2']