def trainDataset(X, Y, val_size, seed):
X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed)
# Spot Check Algorithms
models = []
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
# evaluate each model in turn
results = []
names = []
print('\n')
for name, model in models:
kfold = model_selection.KFold(n_splits=10, random_state=seed)
cv_results = model_selection.cross_val_score(model, X_train, Y_train.values.ravel(), cv=kfold, scoring='accuracy')
results.append(cv_results)
names.append(name)
msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
print(msg)
print('\n')
return results;
def testLogisticRegression(self):
dataset = datasets.load_breast_cancer()
data = dataset.data
target = dataset.target
X_train, X_test, y_train, y_test = train_test_split(data,
target,
test_size=0.2)
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
clf = linear_model.LogisticRegressionCV()
clf.fit(X_train_scaled, y_train)
pred_skl = clf.predict(X_test_scaled)
model = LogisticRegression()
model.fit(X_train_scaled, y_train)
pred_model = model.predict(X_test_scaled)
accuracy_test_skl = accuracy_score(pred_skl, y_test)
accuracy_test_model = accuracy_score(pred_model, y_test)
accuracy_model_skl = accuracy_score(pred_skl, pred_model)
print('scikit accuracy:', accuracy_test_skl)
print('model accuracy:', accuracy_test_model)
print('scikit vs model accuracy: ', accuracy_model_skl)
def __init__(self,n_in,n_hidden_layers,n_hidden_units,n_out):
self.n_hidden_layers=n_hidden_layers;
self.n_hidden_units=n_hidden_units;
self.n_in=n_in;
self.n_out=n_out;
if n_hidden_layers==0:
n_hidden_units=n_in;
self.hiddenLayer = [HiddenLayer(n_in=n_in, n_out=n_hidden_units,activation=sigmoid_stable) for i in range(n_hidden_layers)];
self.logRegressionLayer = LogisticRegression.LogisticRegression(n_hidden_units,n_out);
def __init__(self, csv, variables_indeces, classification_values):
# ---------- ESTRAPOLA DATI ----------
self.values = csv.giveme_values_indeces(variables_indeces, 0)
self.filterY(classification_values[0])
# ---------- MODELLO ----------
self.classification_values = classification_values
self.model = LG.LogisticRegression(self.values) # REGRESSIONE
self.thetas = None
def testPartition(self, traindata, traintarget, validata, valitarget,
index):
model = lr.LogisticRegression()
w = model.fit(traindata,
traintarget,
lr=self.lr,
eps=self.eps,
num_iter=self.iterations)
yp = model.predict(validata, w)
self.valiError[index] = meval.evaluate_acc(yp, valitarget)
def grid_search(df):
valid_size = 0.1
lamb = [('l1', 1e-9), ('l1', 1e-6), ('l1', 1e-3), ('l2', 1e-9),
('l2', 1e-6), ('l2', 1e-3), ('none', 0)]
learning = [1, 0.5, 0.1, 0.01]
adaptiv = ['const', 'decay']
batchsize = [10, 40, 60, 80]
epochs = [1, 10, 40, 60]
mi = 10**4
X, y = parse_data(df, "default payment next month", unbalanced=False)
stats = pd.DataFrame(np.zeros((2 * 2 * 4**3 * 7, 8)),
columns=[
"balanced", "learning rate", "adaptive learning",
"epochs", "batch size", "regularization",
"regularization parameter", "accuracy"
])
index = 0
for balance in [False, True]:
X, y = parse_data(df, "default payment next month", unbalanced=balance)
N = len(y)
print("N = " + str(N) +
"; 0: %.2f; 1: %.2f" % tuple(np.bincount(y) / N))
X_trian, X_eval, y_train, y_eval = train_test_split(
X, y, test_size=valid_size)
for l in lamb:
for gamma in learning:
for a in adaptiv:
for bs in batchsize:
for ep in epochs:
clf_own = Log.LogisticRegression(
max_iter=mi,
mini_batch_size=bs,
epochs=ep,
learning_rate=gamma,
adaptive_learning_rate=a,
regualrization=l,
logging=True)
clf_own.fit(X_trian,
y_train,
split=True,
fraction=valid_size / (1 - valid_size))
d = clf_own.evaluate(X_eval,
y_eval,
data_set="evaluate")
stats.iloc[index] = [
balance, gamma, a, ep, bs, l[0], l[1],
d["accuracy"]
]
index += 1
stats.to_csv("Results/LogReg/hyper_par.csv")
def __init__(self, board):
# Initialize a controller with the (empty) board information,
# and what player it is, either RED or BLACK.
assert playerColor == 1 or playerColor == 2 # RED or BLACK
# Store player, board dimensions.
# don't store actual board, just pass on each move
self.playerColor = playerColor # never used
self.board = board # never used
self.nrows = board.getNumRows()
self.ncols = board.getNumCols() # technically don't need, in board
""" Initialize neural network """
self.num_hidden_units = 50
# The first hidden layer. Feed inputs directly into this.
self.l_hidden = HiddenLayer(board.flattenScaleCurr(),
self.nrows * self.ncols,
self.num_hidden_units)
# The output layer, which is a softmax layer. Note that you only need
# ONE output unit, since given the input (a possible move -->
# some board position), you want to estimate the probability
# of winning.
self.num_output_units = 1
self.l_output = LogisticRegression(self.l_hidden.output,
self.l_hidden.num_output,
self.num_output_units)
# The error functions
self.negative_log_likelihood = self.l_output.negative_log_likelihood
self.errors = self.l_output.errors
# Get parameters
self.params = self.l_hidden.params + self.l_output.params
# import os; cls=lambda: os.system("cls")
# import code; code.interact(local=locals())
# Just a couple of compiled Theano functions for sigmoid and softmax.
# This is for calculating forward pass.
x1 = T.matrix() # some linear combination wx + b
z1 = T.nnet.sigmoid(x1)
self.sigmoid = theano.function([x1], z1)
z2 = T.nnet.softmax(x1) # can probably take out
self.softmax = theano.function([x1], z2)
def test_Logistic_Regression():
iris = datasets.load_iris()
X = iris.data
y = iris.target
X = X[y < 2, :2]
y = y[y < 2]
plt.scatter(X[y == 0, 0], X[y == 0, 1], c='r')
plt.scatter(X[y == 1, 0], X[y == 1, 1], c='b')
plt.show()
X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666)
log_reg = LogisticRegression()
log_reg.fit(X_train, y_train)
print(log_reg.score(X_test, y_test))
print(log_reg.predict_proba(X_test))
print(log_reg.predict(X_test))
print(y_test)
def main():
filename = 'data/default_of_credit_card_clients.xls'
X, y = load_CC_data(filename)
X_train, X_test, y_train, y_test = scale_data_split(X, y)
model = LogisticRegression(n_epochs=1000, size_minibatch=512)
model.fit(X_train, y_train)
pred_model = model.predict(X_test)
# clf = linear_model.LogisticRegressionCV()
# clf.fit(X_train, y_train)
# pred_skl = clf.predict(X_test)
accuracy_model = accuracy_score(pred_model, y_test)
# accuracy_skl = accuracy_score(pred_skl, y_test)
print('accuracy model:', accuracy_model)
def __init__(self, rng, input, n_in, n_out, n_layers, units):
# build and connect each hidden layer
assert n_layers == len(units)
self.hidden_layers = []
self.hidden_layers.append(
HiddenLayer(rng=rng,
input=input,
n_in=n_in,
n_out=units[0],
activation=T.tanh))
i = 1
while i < n_layers:
prev_layer = self.hidden_layers[-1]
layer = HiddenLayer(rng=rng,
input=prev_layer.output,
n_in=units[i - 1],
n_out=units[i],
activation=T.tanh)
self.hidden_layers.append(layer)
i += 1
# logistic regression layer for classification
self.softmax_layer = LogisticRegression(
input=self.hidden_layers[-1].output, n_in=units[-1], n_out=n_out)
self.L1 = sum(
[abs(self.hidden_layers[i].W).sum()
for i in xrange(n_layers)]) + abs(self.softmax_layer.W).sum()
self.L2_sqr = sum([(self.hidden_layers[i].W ** 2).sum() for i in xrange(n_layers)]) + \
(self.softmax_layer.W ** 2).sum()
self.nll = self.softmax_layer.nll
self.errors = self.softmax_layer.errors
self.make_shared_dataset = self.softmax_layer.make_shared_dataset
self.params = self.hidden_layers[0].params
for i in xrange(1, n_layers):
self.params += self.hidden_layers[i].params
self.params += self.softmax_layer.params
self.input = input
self.ypred = self.softmax_layer.ypred
def __testCore(trainingData, testData, optimizer):
lr = LogisticRegression.LogisticRegression(optimizer)
costValue = lr.train(trainingData[:, :-1], trainingData[:, -1])
lr.sigLevel = None
print("theta: {0}, value of cost: {1}".format(lr.theta, costValue))
actualValue = testData[:, -1]
predictValue = (lr.predictValue(testData[:, :-1]) > 0.5) - 0
tp = predictValue[(actualValue == 1).A.flatten(), :].sum()
fp = predictValue[(actualValue == 0).A.flatten(), :].sum()
tn = -(predictValue - 1)[(actualValue == 0).A.flatten(), :].sum()
fn = -(predictValue - 1)[(actualValue == 1).A.flatten(), :].sum()
accuracy = (tp + tn) / (tp + fp + tn + fn)
precision = tp / (tp + fp)
recall = tp / (tp + fn)
f1 = 2 * tp / (2 * tp + fp + fn)
print("accuracy: {0}, precision: {1}, recall: {2}, f1: {3}".format(
accuracy, precision, recall, f1))
def k_fold(data, k):
np.random.shuffle(data)
data_subsets = np.array_split(data, k, axis=0)
avg_acc = 0
start = time.time()
for i in range(k):
training_data = np.concatenate(data_subsets[:i] + data_subsets[i + 1:],
axis=0)
validation_data = data_subsets[i]
logistic_regression_k_fold = LR.LogisticRegression(
training_data[:, 0:-1], training_data[:, -1])
logistic_regression_k_fold.fit(start_learning_rate, end_learning_rate,
gradient_descent_iterations)
y_perdict = logistic_regression_k_fold.predict(validation_data[:,
0:-1])
avg_acc = avg_acc + evaluate_acc(
validation_data[:, -1].reshape(validation_data.shape[0], 1),
y_perdict)
print(avg_acc / k)
print("Logestic average accurrcy: {0:.2%}".format(avg_acc / k))
print("Time cost: ", time.time() - start)
start = time.time()
avg_acc = 0
for i in range(k):
training_data = np.concatenate(data_subsets[:i] + data_subsets[i + 1:],
axis=0)
validation_data = data_subsets[i]
linear_discriminant_analysis_k_fold = LDA.LinearDiscriminantAnalysis(
training_data[:, 0:-1], training_data[:, -1])
linear_discriminant_analysis_k_fold.fit()
y_perdict = linear_discriminant_analysis_k_fold.predict(
validation_data[:, 0:-1])
avg_acc = avg_acc + evaluate_acc(
validation_data[:, -1].reshape(validation_data.shape[0], 1),
y_perdict)
print("LDA average accurrcy: {0:.2%}".format(avg_acc / k))
print("Time cost: ", time.time() - start)
return
def classify():
[
training_data_shared, training_categories_shared,
validation_data_shared, validation_categories_shared, test_data_shared,
test_categories_shared
] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES,
TEST_DIR, NUM_TEST_EXAMPLES)
# create a nn with two 28**2 5x5 convolutional layer (each node looks at 25 pixels in a square) + convolutional pooling layer, then a regular nn layer
print "building the model..."
rng = np.random.RandomState()
nkerns = [20, 50]
batch_size = 100
n_train_batches = NUM_TRAINING_EXAMPLES / batch_size
x = T.matrix('x')
y = T.ivector('y')
index = T.lscalar()
layer0_input = x.reshape((batch_size, 1, 28, 28))
layer0 = ConvPoolingLayer(rng,
input=layer0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2))
# outputs a convoluted image
layer1 = ConvPoolingLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], 12, 12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2))
# outputs a doubly convoluted image
# flatten before putting it through the rest of the NN
layer2_input = layer1.output.flatten(2)
layer2 = HiddenLayer(
rng,
input=layer2_input,
n_in=nkerns[1] * 4 * 4,
n_out=500,
activation=T.tanh) # tanh activation function (sigmoidal)
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
cost = layer3.negative_log_likelihood(y)
test_model = theano.function(
[index],
layer3.errors(y),
givens={
x: test_data_shared[index * batch_size:(index + 1) * batch_size],
y:
test_categories_shared[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x:
validation_data_shared[index * batch_size:(index + 1) *
batch_size],
y:
validation_categories_shared[index * batch_size:(index + 1) *
batch_size]
})
# train on the images
print "training..."
params = layer3.params + layer2.params + layer1.params + layer0.params
grads = T.grad(cost, params) # the magic step
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x:
training_data_shared[index * batch_size:(index + 1) * batch_size],
y:
training_categories_shared[index * batch_size:(index + 1) *
batch_size]
})
epoch = 0
best_loss = 1
while (epoch < N_EPOCHS):
epoch += 1
if epoch % 5 == 0: print "training epoch: ", epoch
if epoch % 5 == 0:
print "validating..."
validation_loss = [
validate_model(i)
for i in range(NUM_VALIDATION_EXAMPLES / batch_size)
]
this_validation_loss = np.mean(validation_loss)
if this_validation_loss < best_loss:
best_loss = this_validation_loss
else:
print "loss is worse now, breaking..."
break
print "loss: ", this_validation_loss * 100, "%"
for minibatch_index in range(n_train_batches):
train_model(minibatch_index)
print "done training! Testing..."
test_losses = [
test_model(i) for i in range(NUM_TEST_EXAMPLES / batch_size)
]
test_score = np.mean(test_losses)
print "Test score loss: ", test_score * 100, "%"
import pandas as pd
import LogisticRegression as lr
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification
X, y = make_classification(n_samples=5000,
n_features=2,
n_redundant=0,
n_informative=1,
n_clusters_per_class=1,
random_state=2)
print(f"This dataset contains {X.shape[0]} entries and {X.shape[1]} features")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
my_regressor = lr.LogisticRegression(X_train, y_train).fit()
sklearn_regressor = LogisticRegression().fit(X_train, y_train)
my_train_accuracy = my_regressor.score()
sklearn_train_accuracy = sklearn_regressor.score(X_train, y_train)
my_test_accuracy = my_regressor.score(X_test, y_test)
sklearn_test_accuracy = sklearn_regressor.score(X_test, y_test)
result = pd.DataFrame([[my_train_accuracy, sklearn_train_accuracy],
[my_test_accuracy, sklearn_test_accuracy]],
['Training Acc.', 'Test Acc.'], ['Our\'s', 'Sklearn\'s'])
print(result)
def test_CNN(dataset='testData4.pkl', params_file='CNN_params7_.pkl'):
dataset = 'testData4.pkl'
datasets = load_data(dataset)
test_set_x, test_set_y = datasets[0]
test_set_x = test_set_x.get_value()
print len(test_set_x)
layer0_params, layer1_params, layer2_params, layer3_params = load_params(
params_file)
x = T.matrix()
label_y = []
for i in range(len(test_set_x)):
if i < 50:
label_y.append(0)
elif i >= 50 and i < 100:
label_y.append(1)
else:
label_y.append(2)
'''print label_y'''
print '... testing the model ...'
# transfrom x from (batchsize, 28*28) to (batchsize,feature,28,28))
# I_shape = (28,28),F_shape = (5,5),
#第一层卷积、池化后 第一层卷积核为20个，每一个样本图片都产生20个特征图，
N_filters_0 = 20
D_features_0 = 1
#输入必须是为四维的，所以需要用到reshape，这一层的输入是一批样本是20个样本，28*28，
layer0_input = x.reshape((50, 1, 40, 36))
layer0 = LeNetConvPoolLayer(input=layer0_input,
params_W=layer0_params[0],
params_b=layer0_params[1],
image_shape=(50, 1, 40, 36),
filter_shape=(N_filters_0, 1, 5, 5),
poolsize=(2, 2))
#layer0.output: (batch_size, N_filters_0, (40-5+1)/2, (36-5+1)/2) -> 20*20*18*16
#卷积之后得到24*24 在经过池化以后得到12*12. 最后输出的格式为20个样本，20个12*12的特征图。卷积操作是对应的窗口呈上一个卷积核参数 相加在求和得到一个特
#征图中的像素点数 这里池化采用最大池化 减少了参数的训练。
N_filters_1 = 50
D_features_1 = N_filters_0
layer1 = LeNetConvPoolLayer(input=layer0.output,
params_W=layer1_params[0],
params_b=layer1_params[1],
image_shape=(50, N_filters_0, 18, 16),
filter_shape=(N_filters_1, D_features_1, 5, 5),
poolsize=(2, 2))
# layer1.output: (20,50,7,6)
#第二层输出为20个样本，每一个样本图片对应着50张4*4的特征图，其中的卷积和池化操作都是同第一层layer0是一样的。
#这一层是将上一层的输出的样本的特征图进行一个平面化，也就是拉成一个一维向量，最后变成一个20*800的矩阵，每一行代表一个样本，
#(20,50,4,4)->(20,(50*4*4))
layer2_input = layer1.output.flatten(2)
#上一层的输出变成了20*800的矩阵，通过全连接，隐层操作，将800变成了500个神经元，里面涉及到全连接。
layer2 = HiddenLayer(layer2_input,
params_W=layer2_params[0],
params_b=layer2_params[1],
n_in=50 * 7 * 6,
n_out=500,
activation=T.tanh)
#这里为逻辑回归层，主要是softmax函数作为输出，
layer3 = LogisticRegression(input=layer2.output,
params_W=layer3_params[0],
params_b=layer3_params[1],
n_in=500,
n_out=3)
##########################
#预测函数
f = theano.function(
[x], #funtion 的输入必须是list，即使只有一个输入
#概率值
layer3.p_y_given_x,
#layer3.y_pred,
allow_input_downcast=True)
#预测的类别pred
pred = f(test_set_x[:])
#将预测的类别pred与真正类别label对比，输出错分的图像
count = 0
for i in range(len(test_set_x)):
print pred[i]
'''
if float(pred[i]) == float(label_y[i]):
count = count + 1
print count
'''
'''
else:
#print('wrong prediction')
#print ('Pred: {0} Actual:{1}'.format(Predict[i], y_test[i]))
wrong = wrong + 1
return (correct / (correct + wrong)) * 100
from sklearn.datasets import load_breast_cancer
cancer = load_breast_cancer()
print(cancer.feature_names)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(cancer.data,
cancer.target,
stratify=cancer.target,
random_state=42)
##Testing the Model
import LogisticRegression as LR
clf = LR.LogisticRegression()
clf.fit(X_train, y_train)
##R_square=clf.R_squared()
##print("R_Square is: ", R_square)
predictions = clf.predict(X_test)
##print("Prediction is: ", predictions)
##print("Real Prediction is: ",y_test)
rounded_predictions = ["{:.2f}".format(value) for value in predictions]
print("Prediction: ", rounded_predictions)
print("Real Prediction is: ", y_test)
print("Accuracy: ", Accuracy(predictions, y_test))
def test_CNN(learning_rate=0.01,
L1_reg=0.00,
L2_reg=0.0001,
n_epochs=1000,
batch_size=20,
n_hidden=500,
dataset='mnist.pkl.gz'):
dataset = load_data(dataset)
#tt = train_set_x.get_value(); tt.shape ---(50000, 784)
train_set_x, train_set_y = dataset[0]
valid_set_x, valid_set_y = dataset[1]
test_set_x, test_set_y = dataset[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
print('training set has %i batches' % n_train_batches)
print('validate set has %i batches' % n_valid_batches)
print('testing set has %i batches' % n_test_batches)
#symbolic variables
x = T.matrix()
y = T.ivector() #lvector: [long int] labels; ivector:[int] labels
minibatch_index = T.lscalar()
print 'build the model...'
rng = numpy.random.RandomState(23455)
# transfrom x from (batchsize, 28*28) to (batchsize,feature,28,28))
# I_shape = (28,28),F_shape = (5,5),
#第一层卷积、池化后 第一层卷积核为20个，每一个样本图片都产生20个特征图，
N_filters_0 = 20
D_features_0 = 1
#输入必须是为四维的，所以需要用到reshape，这一层的输入是一批样本是20个样本，28*28，
layer0_input = x.reshape((batch_size, D_features_0, 28, 28))
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
filter_shape=(N_filters_0, D_features_0, 5, 5),
image_shape=(batch_size, 1, 28, 28))
#layer0.output: (batch_size, N_filters_0, (28-5+1)/2, (28-5+1)/2) -> 20*20*12*12
#卷积之后得到24*24 在经过池化以后得到12*12. 最后输出的格式为20个样本，20个12*12的特征图。卷积操作是对应的窗口呈上一个卷积核参数 相加在求和得到一个特
#征图中的像素点数 这里池化采用最大池化 减少了参数的训练。
N_filters_1 = 50
D_features_1 = N_filters_0
layer1 = LeNetConvPoolLayer(rng,
input=layer0.output,
filter_shape=(N_filters_1, D_features_1, 5, 5),
image_shape=(batch_size, N_filters_0, 12, 12))
# layer1.output: (20,50,4,4)
#第二层输出为20个样本，每一个样本图片对应着50张4*4的特征图，其中的卷积和池化操作都是同第一层layer0是一样的。
#这一层是将上一层的输出的样本的特征图进行一个平面化，也就是拉成一个一维向量，最后变成一个20*800的矩阵，每一行代表一个样本，
#(20,50,4,4)->(20,(50*4*4))
layer2_input = layer1.output.flatten(2)
#上一层的输出变成了20*800的矩阵，通过全连接，隐层操作，将800变成了500个神经元，里面涉及到全连接。
layer2 = HiddenLayer(rng,
layer2_input,
n_in=50 * 4 * 4,
n_out=500,
activation=T.tanh)
#这里为逻辑回归层，主要是softmax函数作为输出，
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
#约束规则
##########################
cost = layer3.negative_log_likelihood(y)
test_model = theano.function(
inputs=[minibatch_index],
outputs=layer3.errors(y),
givens={
x:
test_set_x[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size],
y:
test_set_y[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
})
valid_model = theano.function(
inputs=[minibatch_index],
outputs=layer3.errors(y),
givens={
x:
valid_set_x[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size],
y:
valid_set_y[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
})
params = layer3.params + layer2.params + layer1.params + layer0.params
gparams = T.grad(cost, params)
updates = []
for par, gpar in zip(params, gparams):
updates.append((par, par - learning_rate * gpar))
train_model = theano.function(
inputs=[minibatch_index],
outputs=[cost],
updates=updates,
givens={
x:
train_set_x[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size],
y:
train_set_y[minibatch_index * batch_size:(minibatch_index + 1) *
batch_size]
})
#---------------------Train-----------------------#
print 'training...'
print('training set has %i batches' % n_train_batches)
print('validate set has %i batches' % n_valid_batches)
print('testing set has %i batches' % n_test_batches)
epoch = 0
patience = 10000
patience_increase = 2
validation_frequency = min(n_train_batches, patience / 2)
improvement_threshold = 0.995
best_parameters = None
min_validation_error = numpy.inf
done_looping = False
start_time = time.clock()
while (epoch < n_epochs) and (not done_looping):
epoch += 1
for minibatch_index in xrange(n_train_batches):
#cur_batch_train_error,cur_params = train_model(minibatch_index)
cur_batch_train_error = train_model(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
#validation_error = numpy.mean([valid_model(idx) for idx in xrange(n_valid_batches)])
validation_losses = [
valid_model(i) for i in xrange(n_valid_batches)
]
validation_error = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
validation_error * 100.))
if validation_error < min_validation_error:
if validation_error < min_validation_error * improvement_threshold:
patience = max(patience, iter * patience_increase)
min_validation_error = validation_error
#best_parameters = cur_params
best_iter = iter
save_params(layer0.params, layer1.params, layer2.params,
layer3.params)
test_error = numpy.mean(
[test_model(idx) for idx in xrange(n_test_batches)])
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_error * 100.))
if iter >= patience:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete. Best validation score of %f %% '
'obtained at iteration %i, with test performance %f %%') %
(100 - min_validation_error * 100., best_iter + 1,
100 - test_error * 100.))
print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.7, stratify = y, random_state=0)
print('Size X_train:',X_train.shape)
print('Size X_test:',X_test.shape)
print('Size y_train:',y_train.shape)
print('Size y_test:',y_test.shape)
print('base \tget_ipython().run_line_magic("s", " ratio %2f \ny_train %s ratio %2f \ny_test \t%s ratio %2f'")
% (np.bincount(y), np.bincount(y)[0]/(np.bincount(y)[0]+np.bincount(y)[1]),
np.bincount(y_train),np.bincount(y)[0]/(np.bincount(y)[0]+np.bincount(y)[1]),
np.bincount(y_test), np.bincount(y)[0]/(np.bincount(y)[0]+np.bincount(y)[1])))
# getting params that can be optimized in gridsearch
LogisticRegression().get_params()
solvers = ['newton-cg', 'lbfgs', 'liblinear']
penalty = ['l1', 'l2', 'elasticnet', 'none']
c_values = [100, 10, 1.0, 0.1, 0.01]
# define grid search
grid = dict(solver=solvers,penalty=penalty,C=c_values)
start_time = time.time()
scoring = {'AUC': 'roc_auc'}
gs = GridSearchCV(estimator = LogisticRegression(random_state=42, class_weight={0: 4.0, 1: 0.571}),
param_grid=grid,
scoring=scoring,
## {
## until number of iterations is satisfied
## calculate cost
## adjust parameters
## }
## predict using final parameters
## calculate accuracy
#############################################################################################################################
#################################### Multiclass or 3 class data classification#############################################################
#############################################################################################################################
data,labels = Loaders.readFile("iris-data.txt",True)
m,n = data.shape
# lr1 = LogisticRegression.LogisticRegression(data, labels, 1.0, 8000, regularized=True, normalization = 'l1')
lr1 = LogisticRegression.LogisticRegression(data, labels, 1.0, 8000, regularized=True, normalization = 'l2')
learntParameters, final_costs = lr1.train(data, labels, np.unique(labels))
print 'Number of classes', len(np.unique(labels))
print 'learntParameters(one per class): ',learntParameters
print 'final_costs: ', final_costs
#print len(learntParameters)
classifedLabels = []
for eachData in data:
classifedLabels.append(lr1.classify(eachData, learntParameters))
classifedLabels = np.array(classifedLabels)
# print 'original label', 'classifedLabels'
# for each in zip(labels, classifedLabels):
# print each[0],', ', each[1],', ', each[0]==each[1]
print 'Accuracy on training data: ',(np.sum(classifedLabels == labels)/len(labels))*100,'%'
def sgd_optimization_mnist(dataset):
feats = 784
num_of_train = 50000
iteration_number = 10
learning_rate = 0.01
batch_size = 600
train_set_x, train_set_y = dataset[0]
valid_set_x, valid_set_y = dataset[1]
test_set_x, test_set_y = dataset[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size
#building models
print('building models...')
index = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
classifier = LogisticRegression.LogisticRegression(input=x,
n_in=28 * 28,
n_out=10)
test_model = theano.function(
[index],
classifier.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
valid_model = theano.function(
[index],
classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
g_W = T.grad(cost=classifier.NLL(y), wrt=classifier.W)
g_b = T.grad(cost=classifier.NLL(y), wrt=classifier.b)
train_model = theano.function(
[index],
classifier.NLL(y),
updates=[(classifier.W, classifier.W - learning_rate * g_W),
(classifier.b, classifier.b - learning_rate * g_b)],
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
#model building completed.
best_validation_loss = numpy.inf
#start training model.
start_time = timeit.default_timer()
done_looping = False
epoch = 0
n_epochs = 1000
patience = 5000
patience_increase = 2
improvement_threshold = 0.995
validation_frequency = min(n_train_batches, patience // 2)
test_score = 0
print('start training...')
# index on epoch
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
# index on minibatch_index
for minibatch_index in range(n_train_batches):
#training
minibatch_avg_cost = train_model(minibatch_index)
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
#get Evalid.
validation_losses = [
valid_model(i) for i in range(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error%f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100))
if this_validation_loss < best_validation_loss:
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
test_losses = [
test_model(i) for i in range(n_test_batches)
]
test_score = numpy.mean(test_losses)
print(
'epoch %i, minibatch %i/%i, test error of best model %f %%'
% (epoch, minibatch_index + 1, n_train_batches,
test_score * 100))
with open('best_model.pkl', 'wb') as f:
pickle.dump(classifier, f)
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print(
'Optimization complete with best validation score of %f %%, with test performance %f %%'
% (best_validation_loss * 100, test_score * 100))
print('The code run for %d epochs, with %f epochs/sec' %
(epoch, epoch / (end_time - start_time)))
def main():
#==================================================================================
# ロジスティクス回帰処理
#==================================================================================
print("Start Process1")
#====================================================
# Pre Process（前処理）
#====================================================
#----------------------------------------------------
# read & set data
#----------------------------------------------------
print("reading data")
# scikit-learn ライブラリから iris データを読み込む
iris = datasets.load_iris()
# 3,4 列目の特徴量を抽出し、X_features に保管
X_features = iris.data[:, [2, 3]]
# クラスラベル（教師データ）を取得
y_labels = iris.target
print('Class labels:', numpy.unique(
y_labels)) # ※多くの機械学習ライブラリクラスラベルは文字列から整数にして管理されている（最適な性能を発揮するため）
print("finishing reading data")
#---------------------------------------------------------------------
# トレーニングされたモデルの性能評価を未知のデータで評価するために、
# データセットをトレーニングデータセットとテストデータセットに分割する
#---------------------------------------------------------------------
# scikit-learn の cross_validation モジュールの関数 train_test_split() を用いて、70%:テストデータ, 30%:トレーニングデータに分割
train_test = train_test_split( # 戻り値:list
X_features,
y_labels, #
test_size=0.3, # 0.0~1.0 で指定
random_state=0 #
)
"""
# train_test_split() の戻り値の確認
print("train_test[0]:", train_test[0]) # X_tarin
print("train_test[1]:", train_test[1]) # X_test
print("train_test[2]:", train_test[2]) # y_train
print("train_test[3]:", train_test[3]) # y_test
print("train_test[4]:", train_test[4]) # inavlid value
print("train_test[5]:", train_test[5]) # inavlid value
"""
X_train = train_test[0]
X_test = train_test[1]
y_train = train_test[2]
y_test = train_test[3]
#----------------------------------------------------------------------------------------------------
# scikit-learn の preprocessing モジュールの StandardScaler クラスを用いて、データをスケーリング
#----------------------------------------------------------------------------------------------------
stdScaler = StandardScaler()
# X_train の平均値と標準偏差を計算
stdScaler.fit(X_train)
# 求めた平均値と標準偏差を用いて標準化
X_train_std = stdScaler.transform(X_train)
X_test_std = stdScaler.transform(X_test)
# 分割したデータを行方向に結合（後で plot データ等で使用する）
X_combined_std = numpy.vstack(
(X_train_std, X_test_std)) # list:(X_train_std, X_test_std) で指定
y_combined = numpy.hstack((y_train, y_test))
# 学習データを正規化（後で plot データ等で使用する）
#X_features_std = numpy.copy( X_features ) # ディープコピー（参照コピーではない）
#X_features_std[:,0] = ( X_features[:,0] - X_features[:,0].mean() ) / X_features[:,0].std() # 0列目全てにアクセス[:,0]
#X_features_std[:,1] = ( X_features[:,1] - X_features[:,1].mean() ) / X_features[:,1].std()
#====================================================
# Learning Process (& 説明用の図のpolt)
#====================================================
logReg = LogisticRegression.LogisticRegression()
#plt.subplot(2,1,1) # plt.subplot(行数, 列数, 何番目のプロットか)
logReg.plotSigmoidFunction()
plt.savefig("./LogisticRegression_scikit-learn_1.png", dpi=300)
#plt.subplot(2,1,2) # plt.subplot(行数, 列数, 何番目のプロットか)
logReg.plotCostFunction()
plt.savefig("./LogisticRegression_scikit-learn_2.png", dpi=300)
#plt.show()
logReg.logReg_.fit(X_train_std, y_train)
#====================================================
# 汎化性能の評価
#====================================================
#-------------------------------
# 識別結果＆識別領域の表示
#-------------------------------
MLPlot.drawDiscriminantRegions(X_combined_std,
y_combined,
classifier=logReg.logReg_,
list_test_idx=range(101, 150))
plt.title("Idification Result") #
plt.xlabel("sepal length [Normalized]") # label x-axis
plt.ylabel("petal length [Normalized]") # label y-axis
plt.legend(loc="upper left") # 凡例
plt.tight_layout() # グラフ同士のラベルが重ならない程度にグラフを小さくする。
plt.savefig("./LogisticRegression_scikit-learn_3.png", dpi=300)
#plt.show()
#-------------------------------
# 識別率を計算＆出力
#-------------------------------
y_predict = logReg.logReg_.predict(X_test_std)
# 誤分類のサンプル数を出力
print(
'Misclassified samples: %d' %
(y_test != y_predict).sum()) # %d:10進数, string % data :文字とデータ（値）の置き換え
# 分類の正解率を出力
print('Accuracy: %.2f' % accuracy_score(y_test, y_predict))
#--------------------------------------------------------------------------------------------------------
# predict_proba() 関数を使用して、指定したサンプルのクラスの所属関係を予想
# 戻り値は、サンプルが Iris-Setosa, Iris-Versicolor, Iris-Virginica に所属する確率をこの順で表している.
#--------------------------------------------------------------------------------------------------------
pre0 = logReg.logReg_.predict_proba(X_test_std[0, :].reshape(
1, -1)) # 0番目のテストデータをreshap でタプル化して渡す
pre1 = logReg.logReg_.predict_proba(X_test_std[1, :].reshape(
1, -1)) # 1番目のテストデータをreshap でタプル化して渡す
pre2 = logReg.logReg_.predict_proba(X_test_std[2, :].reshape(
1, -1)) # 2番目のテストデータをreshap でタプル化して渡す
pre3 = logReg.logReg_.predict_proba(X_test_std[3, :].reshape(
1, -1)) # 3番目のテストデータをreshap でタプル化して渡す
#print("predict:", pre0)
#print("predict[0]:", pre0[0])
#print("predict[0,0]:", pre0[0,0]*100)
#print("predict[0,1]:", pre0[0,1]*100)
#print("predict[0,2]:", pre0[0,2]*100)
#------------------------------------------------------------------------
# 各々のサンプルの所属クラスの図示 ["Setosa","Versicolor","Virginica"]
#------------------------------------------------------------------------
# 現在の図をクリア
plt.clf()
# 所属クラスの確率を棒グラフ表示(1,1)
plt.subplot(2, 2, 1) # plt.subplot(行数, 列数, 何番目のプロットか)
plt.title("Probability of class (use predict_proba mesod)")
plt.xlabel("Varieties (Belonging class) [Setosa,Versicolor,Virginica]"
) # label x-axis
plt.ylabel("probability[%]") # label y-axis
plt.ylim(0, 100) # y軸の範囲(0~100)
plt.legend(loc="upper left") # 凡例
# 棒グラフ
plt.bar(left=[0, 1, 2],
height=pre0[0] * 100,
tick_label=["Setosa", "Versicolor", "Virginica"])
plt.tight_layout() # グラフ同士のラベルが重ならない程度にグラフを小さくする。
# 所属クラスの確率を棒グラフ表示(1,2)
plt.subplot(2, 2, 2) # plt.subplot(行数, 列数, 何番目のプロットか)
plt.ylim(0, 100) # y軸の範囲(0~100)
plt.bar(left=[0, 1, 2],
height=pre1[0] * 100,
tick_label=["Setosa", "Versicolor", "Virginica"])
plt.tight_layout() # グラフ同士のラベルが重ならない程度にグラフを小さくする。
# 所属クラスの確率を棒グラフ表示(2,1)
plt.subplot(2, 2, 3) # plt.subplot(行数, 列数, 何番目のプロットか)
plt.ylim(0, 100) # y軸の範囲(0~100)
plt.bar(left=[0, 1, 2],
height=pre2[0] * 100,
tick_label=["Setosa", "Versicolor", "Virginica"])
plt.tight_layout() # グラフ同士のラベルが重ならない程度にグラフを小さくする。
# 所属クラスの確率を棒グラフ表示(2,1)
plt.subplot(2, 2, 4) # plt.subplot(行数, 列数, 何番目のプロットか)
plt.ylim(0, 100) # y軸の範囲(0~100)
plt.bar(left=[0, 1, 2],
height=pre3[0] * 100,
tick_label=["Setosa", "Versicolor", "Virginica"])
plt.tight_layout() # グラフ同士のラベルが重ならない程度にグラフを小さくする。
# 図の保存＆表示
plt.savefig("./LogisticRegression_scikit-learn_4.png", dpi=300)
#plt.show()
#==================================================================================================
# ロジスティクス回帰の逆正則化パラメータ C の値と正則化の強さの関係
# ロジスティクス回帰における、正則化による過学習への対応
#==================================================================================================
print("Start Process2")
weights0 = [] # 重み係数の空リスト（Setosa）を生成
weights1 = [] # 重み係数の空リスト（Versicolor）を生成
weights2 = [] # 重み係数の空リスト（Virginica）を生成
paramesC = [] # 逆正則化パラメータの空リストを生成
# 10個の逆正則化パラメータ C（C=10^-5,C=10^-4, ... C=10, ... , C=10^5 ）に関して、LogisiticReegression オブジェクトを作成し、
# それぞれのモデルを学習データで学習
for c in numpy.arange(-5, 5):
logReg_tmp = LogisticRegression.LogisticRegression(paramC=10.**c)
logReg_tmp.logReg_.fit(X_train_std, y_train)
# 重み係数リストに学習後の重み係数を格納
weights0.append(logReg_tmp.logReg_.coef_[0]
) # coef_[0] : petal length, petal weightの重み (Setosa)
weights1.append(
logReg_tmp.logReg_.coef_[1]
) # coef_[1] : petal length, petal weightの重み (Versicolor)
weights2.append(
logReg_tmp.logReg_.coef_[2]
) # coef_[2] : petal length, petal weightの重み (Virginica)
#print("weight0:", logReg_tmp.logReg_.coef_[0] )
#print("weight1:", logReg_tmp.logReg_.coef_[1] )
#print("weight2:", logReg_tmp.logReg_.coef_[2] )
# 逆正則化パラメータリストにモデルの C 値を格納
paramesC.append(10.**c)
# 重み係数リストを numpy 配列に変換
weights0 = numpy.array(weights0)
weights1 = numpy.array(weights1)
weights2 = numpy.array(weights2)
#----------------------------
# 図の plot
#----------------------------
# 現在の図をクリア
plt.clf()
# Setosa
plt.plot(
paramesC,
weights0[:, 0], # １つ目(petal length)の重み係数 w00
label='petal length (Setosa)',
color="red",
)
plt.plot(
paramesC,
weights0[:, 1], # ２つ目 (petal weight) の重み係数 w01
linestyle='--',
label='petal width (Setosa)',
color="red")
# Versicolor
plt.plot(
paramesC,
weights1[:, 0], # １つ目(petal length)の重み係数 w10
label='petal length (Versicolor)',
color="blue")
plt.plot(
paramesC,
weights1[:, 1], # ２つ目 (petal weight) の重み係数 w11
linestyle='--',
label='petal width (Versicolor)',
color="blue")
# Virginica
plt.plot(
paramesC,
weights2[:, 0], # １つ目(petal length)の重み係数 w20
label='petal length (Virginica)',
color="green")
plt.plot(
paramesC,
weights2[:, 1], # ２つ目 (petal weight) の重み係数 w21
linestyle='--',
label='petal width (Virginica)',
color="green")
plt.ylabel('weight coefficient')
plt.xlabel('C [reverse regularization parameter](=1/λ)')
plt.legend(loc='upper left')
plt.xscale('log') # x軸を対数スケール
plt.tight_layout()
# 図の保存＆表示
plt.savefig("./LogisticRegression_scikit-learn_5.png", dpi=300)
plt.show()
print("Finish main()")
return
import numpy as np import matplotlib.pyplot as plt from sklearn import datasets iris = datasets.load_iris() X = iris.data y = iris.target print(X,y) X = X[y<2,:2] y = y[y<2] X.shape (100, 2) y.shape (100,) plt.scatter(X[y==0,0], X[y==0,1], color="red") plt.scatter(X[y==1,0], X[y==1,1], color="blue") plt.show() from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_test = train_test_split(X, y) import LogisticRegression log_reg = LogisticRegression() log_reg.fit(X_train, y_train) LogisticRegression() log_reg.score(X_test, y_test) log_reg.predict_proba(X_test) log_reg.predict(X_test)
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
import matplotlib.pyplot as plt
import TwoDimensionPlot
##################################模型1########################################
#导入数据集
pima = pd.read_csv('pima-indians-diabetes.csv')
pd0 = pima[pima.columns.tolist()[0:len(pima.columns) - 1]]
pd1 = pima['label']
#归一化
pd0 = pd.DataFrame(preprocessing.scale(pd0))
pima = pd.concat([pd0, pd1], axis=1)
pima_train, pima_test = train_test_split(pima, test_size=0.3)
#定义模型：迭代1000次，学习率默认为0.0001，训练方式随机梯度下降
model_lr1 = LogisticRegression.LogisticRegression(n_iter=1000, gd='sgd')
#训练模型
model_lr1.fit(pima_train)
print("模型1训练集精确率：", model_lr1.accuracy(pima_train))
print("模型1测试集精确率：", model_lr1.accuracy(pima_test))
print("模型1权重为：", model_lr1.weights[0])
#导入数据集
data = pd.read_csv('test.csv')
#划分数据集
data_train, data_test = train_test_split(data, test_size=0.3)
#############################模型2#############################################
##定义模型：迭代10000次，学习率为0.01，训练方式默认批量梯度下降，使用L2正则化
model_lr2 = LogisticRegression.LogisticRegression(eta=0.01,
n_iter=10000,
import numpy as np
from LogisticRegression import *
from SoftmaxRegression import *
import utils
#mnist.init()
X_train, Y_train, X_test, Y_test = mnist.load()
#Slice the training set
X_validation = X_train[0:2000]
Y_validation = Y_train[0:2000]
X_train = X_train[2000:20000, :]
Y_train = Y_train[2000:20000]
X_test = X_test[-2000:, :]
Y_test = Y_test[-2000:]
#Normalize
X_train, X_validation, X_test = utils.normalize(X_train, X_validation, X_test)
# Example of softmax
model = SoftmaxRegression(learningRate=0.01, n=2000, lambd=0.0000001)
model.fit(X_train, Y_train, X_validation, Y_validation, X_test, Y_test)
#Must prepare data for binary classification
X_train, Y_train, X_validation, Y_validation, X_test, Y_test = utils.returnSetsBinaryClassification(
X_train, Y_train, X_validation, Y_validation, X_test, Y_test, 2, 3)
#Example of logistic regression
model = LogisticRegression(learningRate=0.01, n=2000, l2_reg=True, lambd=0.01)
model.fit(X_train, Y_train, X_validation, Y_validation, X_test, Y_test)
...: plt.ylabel('Feature Importance Score')
...: plt.subplots_adjust(bottom=0.3)
...: plt.savefig('1.png')
...: plt.show()
...: # clf.feature_importances_
...: fs = SelectFromModel(selector, prefit=True)
...: X_train = fs.transform(X_train)
...: X_test = fs.transform(X_test)
...: test = fs.transform(test)
In [8]: pivot = int(0.8 * len(X_train))
In [9]: X_train = np.array(X_train)
In [10]: y = np.array(y)
In [11]: validation_features = X_train[pivot:]
In [12]: y_train = np.array(y_train)
In [14]: y_test = np.array(y_test)
In [15]: X_test = np.array(X_test)
In [16]: print("Len validation:", len(X_test),"Len train:", len(X_train))
In [17]: model = LogisticRegression()
In [18]: model.fit(X_train, y_train)
In [19]: test_m_a = model.score(X_test, y_test)
In [20]: print(test_m_a)
In [21]: train_m_a = model.score(X_train, y_train)
In [22]: print(train_m_a)
...: score=accuracy_score(y_test, model.predict(X_test))
...: roc=roc_auc_score(y_test, model.predict_proba(X_test)[:,1])
...: cr=classification_report(y_test, model.predict(X_test))
...: print(score)
...: print(roc)
...: print(cr)
import dataset as ds
import LogisticRegression #user defined module
if __name__ == "__main__":
dataset = ds.Dataset()
lr = LogisticRegression.LogisticRegression()
data = dataset.loadDataFile("pima-indians-diabetes.csv")
dataset.convertToFloat(data)
dataset.normalize(data)
training_Data, test_Data = dataset.splitDataset(data, 0.70)
print('Dataset has {0} rows. Split into training data with {1} rows and test data with {2} rows').format(data.__len__(), training_Data.__len__(), test_Data.__len__())
lr.fitModel(training_Data)
predicted_labels = lr.predict(test_Data)
true_labels = [row[-1] for row in test_Data]
accuracy = lr.accuracyScore(true_labels, predicted_labels)
print("Accuracy of model is {0}%", accuracy)
pass
if __name__ == '__main__':
SetupPath()
from settings import *
from ReadDataSet import *
from LogisticRegression import *
settings.init()
dataset = ReadDataSet()
dataset.Read(GetCurrentPath() + '/Data/mnist.pkl')
[test_data, test_labels] = dataset.getTrainData()
#DisplayDigit(train_data[5], train_labels[5])
import cPickle
with open(GetCurrentPath() + '/Models/weights.pkl', 'rb') as fp:
Wt = cPickle.load(fp)
print shape(Wt)[0]
LR = LogisticRegression(test_data, shape(Wt)[0])
[predict_labels, probs] = LR.test(Wt)
TP = 0
for i in range(0, len(test_labels)):
#print('%d %d %d %f'% (i, test_labels[i],predict_labels[i],probs[i]))
if (test_labels[i] == predict_labels[i]):
TP += 1
acc = 100 * TP / float(len(predict_labels))
print('Recognition Rate = %f' % (acc))
def cnn_mnist(dataset):
learning_rate = 0.1
n_epochs = 200
batch_size = 500
#nkerns : number of kernels on each layer
nkerns = [20, 10]
train_set_x, train_set_y = dataset[0]
valid_set_x, valid_set_y = dataset[1]
test_set_x, test_set_y = dataset[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size
index = T.lscalar()
x = T.matrix('x')
y = T.ivector('y')
print('Building the model...')
layer0_input = x.reshape((batch_size, 1, 28, 28))
layer0 = CNN.ConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2))
layer1 = CNN.ConvPoolLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], 12, 12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2))
layer2_input = layer1.output.flatten(2)
layer2 = MultiLayerPerceptron.HiddenLayer(rng,
input=layer2_input,
n_in=nkerns[1] * 4 * 4,
n_out=80,
activation=T.tanh)
layer3 = LogisticRegression.LogisticRegression(input=layer2.output,
n_in=80,
n_out=10)
cost = layer3.NLL(y)
test_model = theano.function(
[index],
layer3.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
print('1. Test built.')
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
print('2. Valid built.')
params = layer3.params + layer2.params + layer1.params + layer0.params
grads = T.grad(cost, params)
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
print('3. Derivative calculated.')
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
print('4. Train built.')
print('Train model...')
patience = 10000
patience_increase = 2
improvement_threshold = 0.995
validation_frequency = min(n_train_batches, patience // 2)
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in range(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
# if iter % 100 == 0:
# print('training @ iter = ',iter)
cost_ij = train_model(minibatch_index)
if (iter + 1) % validation_frequency == 0:
validation_losses = [
validate_model(i) for i in range(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %.2f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100))
if this_validation_loss < best_validation_loss:
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
test_losses = [
test_model(i) for i in range(n_test_batches)
]
test_score = numpy.mean(test_losses)
print(
'\tepoch %i, minibatch %i/%i, test error of best model %.2f %%'
% (epoch, minibatch_index + 1, n_train_batches,
test_score * 100))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print('Optimization complete')
print(
'Best validation score of %f %% obtained at iteration %i, with test performance %.2f %%'
% (best_validation_loss * 100, best_iter + 1, test_score * 100))
# 10-701 Machine Learning, Spring 2011: Homework 3
# Solution code for question 5 of Section 2
import numpy as np
import LogisticRegression
X_tr = np.loadtxt(open("Data/tr_X.txt", "rb"), delimiter=",")
y_tr = np.loadtxt(open("Data/tr_y.txt", "rb"), delimiter=",")
X_te = np.loadtxt(open("Data/te_X.txt", "rb"), delimiter=",")
y_te = np.loadtxt(open("Data/te_y.txt", "rb"), delimiter=",")
'''
# Let's see the digit represented by this vector
plt.imshow(X_tr[1000].reshape((16,16)))
plt.gray()
plt.show()
'''
model = LogisticRegression.LogisticRegression()
# Pass different values of lambda for solution of 5(b)
model.train_model(X_tr, y_tr, X_te, y_te, lamda=1000)
import time
import BCWDataset, WQDataset
import LDA, LogisticRegression
import KFoldCrossValidator
bcwd = BCWDataset.BCWDataset()
bcwd.load()
wqd = WQDataset.WQDataset()
wqd.load()
print("LDA, BCW")
print(KFoldCrossValidator.validate(LDA.LDA(), 5, bcwd.X, bcwd.y))
print("LogReg, BCW")
print(
KFoldCrossValidator.validate(
LogisticRegression.LogisticRegression(flr=0.6, slr=0.1, num_it=100), 5,
bcwd.X, bcwd.y))
print("LDA, WQ")
print(KFoldCrossValidator.validate(LDA.LDA(), 5, wqd.X, wqd.y))
print("LogReg, WQ")
print(
KFoldCrossValidator.validate(
LogisticRegression.LogisticRegression(flr=0.6, slr=0.1, num_it=100), 5,
wqd.X, wqd.y))