def main_process(train_path, test_path, selected_features):
# 1）get training set data that used to train model
train_data, train_label, m = log.get_train_data_and_label(train_path, selected_features)
print("获取到的数据:", train_data, "----格式为：", train_data.shape)
# 2）initialise the parameter of logistic regression model
theta = log.initialise_logistic_regression_params(train_data) # train_data is X
print("初始化的权值格式", theta.shape)
# 3)train the model to get the parameter theta (theta is a weight matrix)
trained_theta = log.batch_gradient_descent(train_data, train_label, theta, alpha, max_iterations)
print("训练后得到的权值为", trained_theta, "--格式为：", trained_theta.shape)
# 4）get the test set data that used for prediction
test_data, test_id = process_test_set(test_path, selected_features)
print("测试数据为：", test_data, "---格式为：", test_data.shape)
# 5）do prediction
prediction_result_y = log.predict(test_data, trained_theta) # theta_default训练后的theta
print("预测的结果为：", np.mat(prediction_result_y), "格式：", prediction_result_y.shape)
# 6）transform prediction results into submission format
result_list = []
for i in range(len(prediction_result_y)):
# print yPre[i, 0]
if prediction_result_y[i, 0] >= 0.5:
result_list.append(1)
elif prediction_result_y[i, 0] < 0.5:
result_list.append(0)
y_array = np.array(result_list)
return y_array, test_id
def plot_recall_precision(length, features_train, labels_train, features_test, labels_test):
# threshold=[0.1 ,0.2 ,0.3 ,0.4,0.5,0.6,0.7,0.8,0.9]
threshold = [x / 1000.0 for x in range(0, 1001, 1)]
step = length / 3
colors = ['b', 'r', 'g']
for i in range(0, 3):
# ((i+1)*(step)) percent of train data
f = features_train[0:((i + 1) * (step))]
l = labels_train[0:((i + 1) * (step))]
# train classifier for the specific subset of training set
model = LogisticRegression.train(f, l)
# recall-precision for every threshold value
recall = []
precision = []
for t in threshold:
prediction = LogisticRegression.predict(features_test, model, t)
recall.append(measures.recall(labels_test, prediction, 0))
precision.append(measures.precision(labels_test, prediction, 0))
plt.plot(recall, precision, linewidth="2.0", label=str((i + 1) * 33) + "% of train data", color=colors[i])
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Negative tweets')
plt.legend()
plt.show()
def testLogisticRegression(self):
dataset = datasets.load_breast_cancer()
data = dataset.data
target = dataset.target
X_train, X_test, y_train, y_test = train_test_split(data,
target,
test_size=0.2)
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
clf = linear_model.LogisticRegressionCV()
clf.fit(X_train_scaled, y_train)
pred_skl = clf.predict(X_test_scaled)
model = LogisticRegression()
model.fit(X_train_scaled, y_train)
pred_model = model.predict(X_test_scaled)
accuracy_test_skl = accuracy_score(pred_skl, y_test)
accuracy_test_model = accuracy_score(pred_model, y_test)
accuracy_model_skl = accuracy_score(pred_skl, pred_model)
print('scikit accuracy:', accuracy_test_skl)
print('model accuracy:', accuracy_test_model)
print('scikit vs model accuracy: ', accuracy_model_skl)
def trainData3(folder, gradDecent):
datas, labels = LR.loadDataSet(folder, "data3.txt");
datasMap = [];
degree = 6;
m = np.shape(datas)[0];
for k in range(m):
tmp = [1];
for i in range(degree):
for j in range(i + 2):
tmp.append(pow(datas[k][0], i + 1 - j) * pow(datas[k][1], j));
datasMap.append(tmp);
datas = np.mat(datasMap);
alpha = 1;
lamb = 1;
# itemNum = 100; # batch
itemNum = 500;
thetas = gradDecent(datas, labels, alpha, lamb, itemNum);
print "thetas: ", thetas;
thetas = np.mat(thetas);
m = np.shape(datas)[0];
error = 0;
for i in range(m):
pred = LR.classfy(thetas, np.mat(datas[i]));
if pred != labels[i]:
error = error + 1;
errorRate = float(error) * 100 / m;
print "The error rate of the test is %f percents" % errorRate
thetas = np.array(thetas)[0];
Plot.plot(folder, "data3.txt", thetas);
def call_logistic(num_iterations=2000, alpha=0.5, print_cost=False):
'''
Desc : perform logistic regression on given dataset
Input:
1. num_iterations -> num of times to run gradient descent
2. alpha -> learning rate
3. print_cost -> prints the cost function every 100 iterations if set to "True"
Output: Predictions
'''
## Load data for pre-processing
train_set_x_orig, Y_train, test_set_x_orig, Y_test, classes = lr.load_dataset(
)
# flattening the train and test data, converting from 3D to 1D
train_set_x_flatten = train_set_x_orig.reshape(train_set_x_orig.shape[0],
-1).T
test_set_x_flatten = test_set_x_orig.reshape(test_set_x_orig.shape[0],
-1).T
# standerdizing the data
X_train = train_set_x_flatten / 255
X_test = test_set_x_flatten / 255
## building final model
w, b = lr.initialize_parameters(X_train.shape[0])
# fitting best values of w and b on data
parameters, grads, costs = lr.fit(w, b, X_train, Y_train, num_iterations,
alpha, print_cost)
w = parameters["w"]
b = parameters["b"]
Y_prediction_train = lr.predict(w, b, X_train)
Y_prediction_test = lr.predict(w, b, X_test)
print("train accuracy: {} %".format(
100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100))
print("test accuracy: {} %".format(
100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100))
d = {
"costs": costs,
"Y_prediction_test": Y_prediction_test,
"Y_prediction_train": Y_prediction_train,
"w": w,
"b": b,
"alpha": alpha,
"num_iterations": num_iterations
}
return d
def PlotXference_AVG():
global EyeData
global Events
#Change these into arrays and prelocate size
inference = []
noference = []
for idx in range(0, len(Events)):
# inference.append(FindSlices(EyeData[idx], Events[idx], 'Inference', trialTypes))
# noference.append(FindSlices(EyeData[idx], Events[idx], 'Noference', trialTypes))
inference.append(
FindSlices(EyeData[idx], Events[idx], 'Inference', 'typeB', 1))
noference.append(
FindSlices(EyeData[idx], Events[idx], 'Noference', 'typeA', 1))
fig = plt.figure()
fig.suptitle('Gaze X position')
ax = fig.add_subplot(121)
ax2 = fig.add_subplot(122)
for trial in inference:
ax.plot(trial)
ax.set_ylim(0, 2000)
ax.set_ylabel('X coordinate of gaze position')
ax.set_xlabel('Inference trials \n x time course in ms')
for trial in noference:
ax2.plot(trial)
ax2.set_ylim(0, 2000)
ax2.set_xlabel('No inference trials \n x time course in ms')
ticks = ax.get_xticks() * 16
ax.set_xticklabels(ticks.astype(int))
ax2.set_xticklabels(ticks.astype(int))
inf_cat = [1 for i in range(1, len(inference) + 1)]
nof_cat = [0 for i in range(1, len(noference) + 1)]
known_cat = np.hstack((np.array(inf_cat), np.array(nof_cat)))
ferences = np.vstack((inference, noference))
PlotAverage_X(np.array(inference), np.array(noference))
#
components = PCA.myPCA(ferences, known_cat)
components = components * 1000
LOG_REG.logReg(known_cat, components)
#components_tmp = components *1000
np.savetxt("eda_pcaResults.csv",
np.hstack((known_cat.reshape(len(known_cat), 1), components)),
delimiter=",")
def colicTest():
'''
对数据格式化处理
'''
frTrain = open('horseColicTraining.txt')
frTest = open('horseColicTest.txt')
trainingSet = []
trainingLabels = []
for line in frTrain.readlines():
currLine = line.strip().split('\t')
lineArr = []
for i in range(21):
lineArr.append(float(currLine[i]))
trainingSet.append(lineArr)
trainingLabels.append(float(currLine[21]))
trainWeights = LR.stocGradAscent1(array(trainingSet), trainingLabels, 500)
errorCount = 0
numTestVec = 0.0
#读取测试集数据，计算分类错误的样本条数和最终的错误率
for line in frTest.readlines():
numTestVec += 1.0
currLine = line.strip().split('\t')
lineArr = []
for i in range(21):
lineArr.append(float(currLine[i]))
if int(classifyVector(array(lineArr), trainWeights)) != int(
currLine[21]):
errorCount += 1
errorRate = (float(errorCount) / numTestVec)
print("the error rate is: %f" % errorRate)
return errorRate
def main():
X, y = generateData()
start = timer()
test_x, test_y = generateData(1992)
clf = nn.multilayerperceptron(layers=[
nn.layer("tanh", 3),
nn.layer("tanh", 5),
nn.layer("tanh", 5),
nn.layer("softmax", 2)
],
opt_function='momentum',
drop_out=1)
clf = lr.logisticregression()
#clf = linear_model.LogisticRegression(solver="sag")
yt = pd.get_dummies(y, prefix='class').values
tyt = pd.get_dummies(test_y, prefix='class').values
clf.fit(test_x, test_y)
print(clf.coef_)
timeusage = timer() - start
print("%f seconds" % timeusage)
y_ = clf.predict(X)
count = 0
for i in range(len(y)):
if (y[i] == y_[i]): count += 1
print(count / len(y))
# Plot the decision boundary
plot_decision_boundary(lambda x: clf.predict(x), X, y)
plt.title("Multilayer Perceptron")
plt.show()
def colicTest():
frTrain = open('horseColicTraining.txt'); frTest = open('horseColicTest.txt')
trainingSet = []; trainingLabels = []
for line in frTrain.readlines():
currLine = line.strip().split('\t')
lineArr =[]
for i in range(21):
lineArr.append(float(currLine[i]))
trainingSet.append(lineArr)
trainingLabels.append(float(currLine[21]))
'''计算回归系数向量这里可以自由设定迭代的次数，例如在训练集
上使用500次迭代，实验结果表明这比默认迭代150次的效果更好。'''
trainWeights = LogisticRegression.stocGranAscentImprove(array(trainingSet), trainingLabels, 1000)
'''在系数计算完成之后，导人测试集并计算分类错误率'''
errorCount = 0; numTestVec = 0.0
for line in frTest.readlines():
numTestVec += 1.0
currLine = line.strip().split('\t')
lineArr =[]
for i in range(21):
lineArr.append(float(currLine[i]))
if int(classifyVector(array(lineArr), trainWeights))!= int(currLine[21]):
errorCount += 1
errorRate = (float(errorCount)/numTestVec)
print "the error rate of this test is: %f" % errorRate
return errorRate
def trainDataset(X, Y, val_size, seed):
X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed)
# Spot Check Algorithms
models = []
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
# evaluate each model in turn
results = []
names = []
print('\n')
for name, model in models:
kfold = model_selection.KFold(n_splits=10, random_state=seed)
cv_results = model_selection.cross_val_score(model, X_train, Y_train.values.ravel(), cv=kfold, scoring='accuracy')
results.append(cv_results)
names.append(name)
msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
print(msg)
print('\n')
return results;
def C_comparison(length, features_train, labels_train, features_test, labels_test):
C = [0.001, 0.05, 0.1, 0.3, 0.5, 0.8, 1, 10, 100, 350, 500, 1000, 3500, 5000, 10000, 50000, 100000]
scores = []
for c in C:
model = LogisticRegression.train(features_train, labels_train, c)
prediction = LogisticRegression.predict(features_test, model)
scores.append((measures.avgF1(labels_test, prediction, 0, 1)))
plt.plot(C, scores, color="blue", linewidth="2.0")
plt.xticks(C)
plt.ylabel("F1")
plt.xlabel("C")
plt.show()
def test_analyze_logreg(self):
X, y = gen_logistic_data()
solver = logReg.LogisticRegressionSolver()
with CapturedStdout():
analyzerResults = analyze(solver,
X,
y,
optimizationParams={
"nnTopology": "",
"Lambda": 1,
"functions": None
},
iterations=40,
bins=3,
tries=4,
sample_iterations=4)
npt.assert_equal(
analyzerResults.sampleCountAnalyzis.sampleCount,
[4, 6, 8, 14, 20, 38, 56, 112, 168, 334, 500, 1000, 1500])
npt.assert_almost_equal(
analyzerResults.sampleCountAnalyzis.errorTrain,
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], 2)
npt.assert_almost_equal(analyzerResults.sampleCountAnalyzis.errorCV, [
0.0646667, 0.104, 0.1846667, 0.0126667, 0.0166667, 0.0053333,
0.0006667, 0.0006667, 0., 0.0006667, 0., 0., 0.
])
npt.assert_equal(analyzerResults.iterationCountAnalyzis.iterationCount,
[2, 4, 6, 10, 14, 27, 40])
npt.assert_almost_equal(
analyzerResults.iterationCountAnalyzis.errorTrain,
[0., 0.0005, 0., 0., 0., 0., 0.], 5)
npt.assert_almost_equal(analyzerResults.iterationCountAnalyzis.errorCV,
[0, 0, 0, 0, 0, 0, 0], 5)
def main():
X, y = generateData()
start = timer()
test_x,test_y = generateData(1992)
clf = nn.multilayerperceptron(layers=[
nn.layer("tanh",3),
nn.layer("tanh",5),
nn.layer("tanh",5),
nn.layer("softmax",2)
],opt_function='momentum',drop_out=1)
clf = lr.logisticregression()
#clf = linear_model.LogisticRegression(solver="sag")
yt = pd.get_dummies(y,prefix='class').values
tyt = pd.get_dummies(test_y,prefix='class').values
clf.fit(test_x, test_y)
print(clf.coef_ )
timeusage = timer() - start
print("%f seconds"%timeusage)
y_ = clf.predict(X)
count = 0;
for i in range(len(y)):
if (y[i] == y_[i]): count += 1
print(count / len(y))
# Plot the decision boundary
plot_decision_boundary(lambda x: clf.predict(x),X,y)
plt.title("Multilayer Perceptron")
plt.show()
def plotFeaturesF1(features_train, labels_train, features_test, labels_test):
x = list(np.arange(len(features_train[0])))
# x = list(np.arange(5))
y = []
for i in range(0, len(features_train[0])):
f_train = features_train[:, i]
f_test = features_test[:, i]
f_train = f_train.reshape(f_train.shape[0], 1)
f_test = f_test.reshape(f_test.shape[0], 1)
model = LogisticRegression.train(f_train, labels_train)
prediction = LogisticRegression.predict(f_test, model)
y.append(measures.avgF1(labels_test, prediction, 0, 1))
plt.plot(x, y, color="blue", linewidth="2.0")
plt.ylabel("F1")
plt.xlabel("# of Feature")
plt.xticks(x)
plt.show()
def __init__(self,n_in,n_hidden_layers,n_hidden_units,n_out):
self.n_hidden_layers=n_hidden_layers;
self.n_hidden_units=n_hidden_units;
self.n_in=n_in;
self.n_out=n_out;
if n_hidden_layers==0:
n_hidden_units=n_in;
self.hiddenLayer = [HiddenLayer(n_in=n_in, n_out=n_hidden_units,activation=sigmoid_stable) for i in range(n_hidden_layers)];
self.logRegressionLayer = LogisticRegression.LogisticRegression(n_hidden_units,n_out);
def __init__(self, csv, variables_indeces, classification_values):
# ---------- ESTRAPOLA DATI ----------
self.values = csv.giveme_values_indeces(variables_indeces, 0)
self.filterY(classification_values[0])
# ---------- MODELLO ----------
self.classification_values = classification_values
self.model = LG.LogisticRegression(self.values) # REGRESSIONE
self.thetas = None
def testOne(self):
X = np.array([[8, 1, 6], [3, 5, 7], [4, 9, 2], [8, 1, 6], [3, 5, 7], [4, 9, 2]])
y = np.array([[1], [0], [1], [0], [1], [0]])
theta = np.array([[0], [1], [0]])
expectedJ = 2.6067
expectedTheta = np.array([[1.7760], [2.3988], [1.9464]])
[actualJ, actualTheta] = LR.costFunction(X, y, theta)
np.testing.assert_almost_equal(actualJ, expectedJ, decimal=4)
np.testing.assert_almost_equal(actualTheta, expectedTheta, decimal=4)
def PlotXference_AVG():
global EyeData
global Events
# Change these into arrays and prelocate size
inference = []
noference = []
for idx in range(0, len(Events)):
# inference.append(FindSlices(EyeData[idx], Events[idx], 'Inference', trialTypes))
# noference.append(FindSlices(EyeData[idx], Events[idx], 'Noference', trialTypes))
inference.append(FindSlices(EyeData[idx], Events[idx], "Inference", "typeB", 1))
noference.append(FindSlices(EyeData[idx], Events[idx], "Noference", "typeA", 1))
fig = plt.figure()
fig.suptitle("Gaze X position")
ax = fig.add_subplot(121)
ax2 = fig.add_subplot(122)
for trial in inference:
ax.plot(trial)
ax.set_ylim(0, 2000)
ax.set_ylabel("X coordinate of gaze position")
ax.set_xlabel("Inference trials \n x time course in ms")
for trial in noference:
ax2.plot(trial)
ax2.set_ylim(0, 2000)
ax2.set_xlabel("No inference trials \n x time course in ms")
ticks = ax.get_xticks() * 16
ax.set_xticklabels(ticks.astype(int))
ax2.set_xticklabels(ticks.astype(int))
inf_cat = [1 for i in range(1, len(inference) + 1)]
nof_cat = [0 for i in range(1, len(noference) + 1)]
known_cat = np.hstack((np.array(inf_cat), np.array(nof_cat)))
ferences = np.vstack((inference, noference))
PlotAverage_X(np.array(inference), np.array(noference))
#
components = PCA.myPCA(ferences, known_cat)
components = components * 1000
LOG_REG.logReg(known_cat, components)
# components_tmp = components *1000
np.savetxt("eda_pcaResults.csv", np.hstack((known_cat.reshape(len(known_cat), 1), components)), delimiter=",")
def testPartition(self, traindata, traintarget, validata, valitarget,
index):
model = lr.LogisticRegression()
w = model.fit(traindata,
traintarget,
lr=self.lr,
eps=self.eps,
num_iter=self.iterations)
yp = model.predict(validata, w)
self.valiError[index] = meval.evaluate_acc(yp, valitarget)
def main():
filename = 'data/default_of_credit_card_clients.xls'
X, y = load_CC_data(filename)
X_train, X_test, y_train, y_test = scale_data_split(X, y)
model = LogisticRegression(n_epochs=1000, size_minibatch=512)
model.fit(X_train, y_train)
pred_model = model.predict(X_test)
# clf = linear_model.LogisticRegressionCV()
# clf.fit(X_train, y_train)
# pred_skl = clf.predict(X_test)
accuracy_model = accuracy_score(pred_model, y_test)
# accuracy_skl = accuracy_score(pred_skl, y_test)
print('accuracy model:', accuracy_model)
def crossValidate(data, labels, chunks, dataCont=None):
''' Perform n-fold cross validation.
Given data array, labels, the chunks as folds, the function performs
cross validation by using 4 out of 5 folds as training and the fifth as
testing set. This is repeated 5 times with a different chunk of the fold
serving as testing set. With each fold, both NB and logistic regression
is fitted. The validation error with both algorithms are returned.
'''
errLG = np.empty(len(chunks)) # pred error, logistic regression
nIters = np.empty(len(chunks),
int) # number of iterations for logistic reg
errNB = np.empty(len(chunks)) # pred error, naive Bayes
for ck in range(len(chunks)):
# get index and dataset for current fold of cross-validation
trnIdx = np.hstack([x for n, x in enumerate(chunks) if n != ck])
vldIdx = np.hstack([x for n, x in enumerate(chunks) if n == ck])
dataTrain, labelTrain = data[trnIdx], labels[trnIdx] # training
dataTest, labelTest = data[vldIdx], labels[vldIdx] # validation
if labels.ndim > 1: # if labels have more than one dimension
labelTest = labelTest.argmax(axis=1) # collapse to 1D
## Fit and predict with naive Bayes
prb = NB.NB_Train(dataTrain, labelTrain)
predNB = NB.NB_Pred(dataTest, prb)
errNB[ck] = errRate(predNB, labelTest) # error with naive Bayes
## Fit and predict with logistic regression
if dataCont is not None: # if given non-discretized data
dataTrain = dataCont[trnIdx]
dataTest = dataCont[vldIdx]
if labels.ndim > 1: # if more than two classes, use multinomial logistic
wts, nIters[ck] = LG.fitLogisticNK(dataTrain, labelTrain, 0.5)
predLG = LG.predLogisticNK(dataTest, wts)
else: # binary response var, use regular logistic regression
wt, nIters[ck] = LG.fitLogisticReg(dataTrain, labelTrain, 0.5)
predLG = LG.predLogistic(dataTest, wt)
errLG[ck] = errRate(predLG, labelTest) # error with logistic reg
return errLG, errNB, nIters
def classifyVector(inX, weights):
'''
LogisticRegression分类
inX:特征向量
weights:回归系数
'''
prob = LR.sigmoid(sum(inX * weights))
if prob > 0.5:
return 1.0
else:
return 0.0
def test_test(self):
data = [
(SparseVector.SparseVector({'a': -1, 'b': -1}), -1),
(SparseVector.SparseVector({'a': -1, 'b': -1}), 1),
(SparseVector.SparseVector({'a':1, 'b': 1}), 1),
(SparseVector.SparseVector({'a':1, 'b': 1}), 1),
(SparseVector.SparseVector({'a':1, 'b': 1}), 1),
(SparseVector.SparseVector({'a':1, 'b': 1}), 1)
]
beta = SparseVector.SparseVector({'a': 2, 'b': 2})
scores = LogisticRegression.test(data, beta)
def trainData2(folder, gradDecent):
datas, labels = LR.loadDataSet(folder, "data2.txt");
alpha = 0.1;
lamb = 1;
# itemNum = 500; # batch
itemNum = 100;
thetas = gradDecent(datas, labels, alpha, lamb, itemNum);
print "thetas: ", thetas;
thetas = np.mat(thetas);
m = np.shape(datas)[0];
error = 0;
for i in range(m):
pred = LR.classfy(thetas, np.mat(datas[i]));
if pred != labels[i]:
error = error + 1;
errorRate = float(error) * 100 / m;
print "The error rate of the test is %f percents" % errorRate
thetas = np.array(thetas)[0];
Plot.plot(folder, "data2.txt", thetas);
def plotBolder(theta, data, label_x, label_y, label_pos, label_neg, axes=None):
plt.scatter(45, 85, s=60, c='r', marker='v', label='(45, 85)')
plotData(data, label_x, label_y, label_pos, label_neg, axes)
X = np.c_[np.ones((data.shape[0], 1)), data[:, :2]]
x1_min, x1_max = X[:, 1].min(), X[:, 1].max(),
x2_min, x2_max = X[:, 2].min(), X[:, 2].max(),
xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max),
np.linspace(x2_min, x2_max))
h = LR.sigmoid(np.c_[np.ones((xx1.ravel().shape[0], 1)),
xx1.ravel(),
xx2.ravel()].dot(theta))
h = h.reshape(xx1.shape)
plt.contour(xx1, xx2, h, [0.5], linewidths=1, colors='b')
def grid_search(df):
valid_size = 0.1
lamb = [('l1', 1e-9), ('l1', 1e-6), ('l1', 1e-3), ('l2', 1e-9),
('l2', 1e-6), ('l2', 1e-3), ('none', 0)]
learning = [1, 0.5, 0.1, 0.01]
adaptiv = ['const', 'decay']
batchsize = [10, 40, 60, 80]
epochs = [1, 10, 40, 60]
mi = 10**4
X, y = parse_data(df, "default payment next month", unbalanced=False)
stats = pd.DataFrame(np.zeros((2 * 2 * 4**3 * 7, 8)),
columns=[
"balanced", "learning rate", "adaptive learning",
"epochs", "batch size", "regularization",
"regularization parameter", "accuracy"
])
index = 0
for balance in [False, True]:
X, y = parse_data(df, "default payment next month", unbalanced=balance)
N = len(y)
print("N = " + str(N) +
"; 0: %.2f; 1: %.2f" % tuple(np.bincount(y) / N))
X_trian, X_eval, y_train, y_eval = train_test_split(
X, y, test_size=valid_size)
for l in lamb:
for gamma in learning:
for a in adaptiv:
for bs in batchsize:
for ep in epochs:
clf_own = Log.LogisticRegression(
max_iter=mi,
mini_batch_size=bs,
epochs=ep,
learning_rate=gamma,
adaptive_learning_rate=a,
regualrization=l,
logging=True)
clf_own.fit(X_trian,
y_train,
split=True,
fraction=valid_size / (1 - valid_size))
d = clf_own.evaluate(X_eval,
y_eval,
data_set="evaluate")
stats.iloc[index] = [
balance, gamma, a, ep, bs, l[0], l[1],
d["accuracy"]
]
index += 1
stats.to_csv("Results/LogReg/hyper_par.csv")
def __init__(self, board):
# Initialize a controller with the (empty) board information,
# and what player it is, either RED or BLACK.
assert playerColor == 1 or playerColor == 2 # RED or BLACK
# Store player, board dimensions.
# don't store actual board, just pass on each move
self.playerColor = playerColor # never used
self.board = board # never used
self.nrows = board.getNumRows()
self.ncols = board.getNumCols() # technically don't need, in board
""" Initialize neural network """
self.num_hidden_units = 50
# The first hidden layer. Feed inputs directly into this.
self.l_hidden = HiddenLayer(board.flattenScaleCurr(),
self.nrows * self.ncols,
self.num_hidden_units)
# The output layer, which is a softmax layer. Note that you only need
# ONE output unit, since given the input (a possible move -->
# some board position), you want to estimate the probability
# of winning.
self.num_output_units = 1
self.l_output = LogisticRegression(self.l_hidden.output,
self.l_hidden.num_output,
self.num_output_units)
# The error functions
self.negative_log_likelihood = self.l_output.negative_log_likelihood
self.errors = self.l_output.errors
# Get parameters
self.params = self.l_hidden.params + self.l_output.params
# import os; cls=lambda: os.system("cls")
# import code; code.interact(local=locals())
# Just a couple of compiled Theano functions for sigmoid and softmax.
# This is for calculating forward pass.
x1 = T.matrix() # some linear combination wx + b
z1 = T.nnet.sigmoid(x1)
self.sigmoid = theano.function([x1], z1)
z2 = T.nnet.softmax(x1) # can probably take out
self.softmax = theano.function([x1], z2)
def train_lr(learning_rate=9e-2, l2reg=0.0, batch_size=500, n_epochs=5000, outfile='outputs.csv'):
'''
Training Otto data using Logistic Regression
Create by Ghifar
Tuesday, 14/04/2015
'''
print '[train_lr] learning_rate : ',learning_rate
print ' l2reg : ',l2reg
print ' batch_size : ',batch_size
print ' n_epochs : ',n_epochs
# Load training set (with contrast normalization preprocessing)
# (x_train, y_train), mu, std = load_otto_data(filepath='train.csv')
# (x_train, y_train), mu, std = load_otto_data(filepath='train.csv', preprocess='cn')
(x_train, y_train), mu, std = load_otto_data(filepath='train.csv', preprocess='0-1')
# # Load test set
x_test=None
y_test=None
# (x_test, y_test), mu, std = load_otto_data(filepath='test.csv',preprocess='cn',mu=mu, std=std)
(x_test, y_test), mu, std = load_otto_data(filepath='test.csv', preprocess='0-1')
# print y_test.eval()
# Contruct and compile LR model
n_in = 93
n_out = 9
print x_train.eval().shape
print y_train.eval().shape
lr = LogisticRegression(n_in=n_in, n_out=n_out)
lr.compile(
train_set_x=x_train, train_set_y = y_train, #training set
valid_set_x=x_train, valid_set_y = y_train, #validation set
test_set_x=x_test, # test set
learning_rate=learning_rate,
batch_size=batch_size
)
# Training with SGD
lr.training_sgd(n_epochs=n_epochs)
# predict
print 'Predict the label...'
y_predict = lr.test_model(x_test.eval())
print 'Save the prediciton: ',outfile
write_otto_output(y_predict,filename=outfile)
print '[train_lr] learning_rate : ',learning_rate
print ' l2reg : ',l2reg
print ' batch_size : ',batch_size
print ' n_epochs : ',n_epochs
def test_Logistic_Regression():
iris = datasets.load_iris()
X = iris.data
y = iris.target
X = X[y < 2, :2]
y = y[y < 2]
plt.scatter(X[y == 0, 0], X[y == 0, 1], c='r')
plt.scatter(X[y == 1, 0], X[y == 1, 1], c='b')
plt.show()
X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666)
log_reg = LogisticRegression()
log_reg.fit(X_train, y_train)
print(log_reg.score(X_test, y_test))
print(log_reg.predict_proba(X_test))
print(log_reg.predict(X_test))
print(y_test)
def testTwo(self):
X = np.array(
[
[1, 1, 1, 1],
[1, 1, 1, 1],
[1, 1, 1, 1],
[1, 1, 1, 1],
[16, 2, 3, 13],
[5, 11, 10, 8],
[9, 7, 6, 12],
[4, 14, 15, 1],
]
)
y = np.array([[1], [0], [1], [0], [1], [0], [1], [0]])
theta = np.array([[0], [1], [0], [1]])
expectedJ = 4.8135
expectedTheta = np.array([[1.3154], [3.3154], [3.3154], [1.3154]])
[actualJ, actualTheta] = LR.costFunction(X, y, theta)
np.testing.assert_almost_equal(actualJ, expectedJ, decimal=4)
np.testing.assert_almost_equal(actualTheta, expectedTheta, decimal=4)
def __init__(self, rng, input, n_in, n_out, n_layers, units):
# build and connect each hidden layer
assert n_layers == len(units)
self.hidden_layers = []
self.hidden_layers.append(
HiddenLayer(rng=rng,
input=input,
n_in=n_in,
n_out=units[0],
activation=T.tanh))
i = 1
while i < n_layers:
prev_layer = self.hidden_layers[-1]
layer = HiddenLayer(rng=rng,
input=prev_layer.output,
n_in=units[i - 1],
n_out=units[i],
activation=T.tanh)
self.hidden_layers.append(layer)
i += 1
# logistic regression layer for classification
self.softmax_layer = LogisticRegression(
input=self.hidden_layers[-1].output, n_in=units[-1], n_out=n_out)
self.L1 = sum(
[abs(self.hidden_layers[i].W).sum()
for i in xrange(n_layers)]) + abs(self.softmax_layer.W).sum()
self.L2_sqr = sum([(self.hidden_layers[i].W ** 2).sum() for i in xrange(n_layers)]) + \
(self.softmax_layer.W ** 2).sum()
self.nll = self.softmax_layer.nll
self.errors = self.softmax_layer.errors
self.make_shared_dataset = self.softmax_layer.make_shared_dataset
self.params = self.hidden_layers[0].params
for i in xrange(1, n_layers):
self.params += self.hidden_layers[i].params
self.params += self.softmax_layer.params
self.input = input
self.ypred = self.softmax_layer.ypred
def __testCore(trainingData, testData, optimizer):
lr = LogisticRegression.LogisticRegression(optimizer)
costValue = lr.train(trainingData[:, :-1], trainingData[:, -1])
lr.sigLevel = None
print("theta: {0}, value of cost: {1}".format(lr.theta, costValue))
actualValue = testData[:, -1]
predictValue = (lr.predictValue(testData[:, :-1]) > 0.5) - 0
tp = predictValue[(actualValue == 1).A.flatten(), :].sum()
fp = predictValue[(actualValue == 0).A.flatten(), :].sum()
tn = -(predictValue - 1)[(actualValue == 0).A.flatten(), :].sum()
fn = -(predictValue - 1)[(actualValue == 1).A.flatten(), :].sum()
accuracy = (tp + tn) / (tp + fp + tn + fn)
precision = tp / (tp + fp)
recall = tp / (tp + fn)
f1 = 2 * tp / (2 * tp + fp + fn)
print("accuracy: {0}, precision: {1}, recall: {2}, f1: {3}".format(
accuracy, precision, recall, f1))
def k_fold(data, k):
np.random.shuffle(data)
data_subsets = np.array_split(data, k, axis=0)
avg_acc = 0
start = time.time()
for i in range(k):
training_data = np.concatenate(data_subsets[:i] + data_subsets[i + 1:],
axis=0)
validation_data = data_subsets[i]
logistic_regression_k_fold = LR.LogisticRegression(
training_data[:, 0:-1], training_data[:, -1])
logistic_regression_k_fold.fit(start_learning_rate, end_learning_rate,
gradient_descent_iterations)
y_perdict = logistic_regression_k_fold.predict(validation_data[:,
0:-1])
avg_acc = avg_acc + evaluate_acc(
validation_data[:, -1].reshape(validation_data.shape[0], 1),
y_perdict)
print(avg_acc / k)
print("Logestic average accurrcy: {0:.2%}".format(avg_acc / k))
print("Time cost: ", time.time() - start)
start = time.time()
avg_acc = 0
for i in range(k):
training_data = np.concatenate(data_subsets[:i] + data_subsets[i + 1:],
axis=0)
validation_data = data_subsets[i]
linear_discriminant_analysis_k_fold = LDA.LinearDiscriminantAnalysis(
training_data[:, 0:-1], training_data[:, -1])
linear_discriminant_analysis_k_fold.fit()
y_perdict = linear_discriminant_analysis_k_fold.predict(
validation_data[:, 0:-1])
avg_acc = avg_acc + evaluate_acc(
validation_data[:, -1].reshape(validation_data.shape[0], 1),
y_perdict)
print("LDA average accurrcy: {0:.2%}".format(avg_acc / k))
print("Time cost: ", time.time() - start)
return
def __init__(self,
norm_type="Normalization",
iterations=5,
base_classifier="SVM"):
super().__init__()
self.iterations = iterations
self.norm_type = norm_type
# self.prediction = None
# self.probability = None
self.classifier_set = None
if base_classifier == "SVM":
self.base_classifier = SVM.SVMClassifier()
elif base_classifier == "KNN":
self.base_classifier = KNN.KNNClassifier()
elif base_classifier == "DecisionTree":
self.base_classifier = DecisionTree.DecisionTreeClassifier()
elif base_classifier == "Logistic":
self.base_classifier = LogisticRegression.LogisticRegressionClassifier(
)
elif base_classifier == "Perceptron":
self.base_classifier = Perceptron.PerceptronClassifier()
super().__init__()
def buildAdv():
rng = np.random.RandomState()
print "Loading Data..."
[training_data_shared, training_categories_shared, validation_data_shared, validation_categories_shared, test_data_shared, test_categories_shared] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES, TEST_DIR, NUM_TEST_EXAMPLES)
current_generated_data_shared = theano.shared(np.zeros_like(training_data_shared.get_value()))
current_generated_categories_shared = theano.shared(np.int32(np.zeros_like(training_categories_shared.get_value())))
print "Building the model..."
# we want a generative mlp that goes from a category and a random seed, through three hidden layers, to a 28x28 image
xg_category = T.cast(T.ivector('xg_category'), 'int32') # input category
xg_seed = T.matrix('xg_seed')
xg_category_onehot = T.extra_ops.to_one_hot(xg_category, 10)
# first mlp hidden layer """"""
glayer0 = HiddenLayer(rng, input=T.concatenate((xg_category_onehot, xg_seed), axis=1), n_in=20, n_out=100, activation=T.tanh)
# second mlp hidden layer
glayer1 = HiddenLayer(rng, input=glayer0.output, n_in=100, n_out=500, activation=T.tanh)
glayer2 = HiddenLayer(rng, input=glayer1.output, n_in=500, n_out=1000, activation=T.tanh)
# last mlp hidden layer
glayer3 = HiddenLayer(rng, input=glayer2.output, n_in=1000, n_out=28*28, activation=T.tanh)
# the output of the generative network
goutput = glayer3.output.reshape((batch_size, 1, 28,28))
# we want a discriminative mlp that goes from an image, through a convolutional layer, through a hidden layer, through a logistic regression to decide whether
xd = glayer3.output # input images
yd = xg_category # label for the images
yd_onehot = T.extra_ops.to_one_hot(yd, 10)
zd = T.ivector('z') # real or not
dlayer0Input = xd.reshape((batch_size, 1, 28, 28)) # reassemble the image
# convolutional layer
dlayer0 = ConvPoolingLayer(rng, input=dlayer0Input, image_shape=(batch_size, 1, 28, 28), filter_shape=(20, 1, 5, 5), poolsize=(2,2))
# first mlp hidden layer
dlayer1 = HiddenLayer(rng, input=dlayer0.output.flatten(2), n_in=20*9*4*4, n_out=500, activation=T.tanh)
# second mlp hidden layer
dlayer2 = HiddenLayer(rng, input=T.concatenate((dlayer1.output, yd_onehot), axis=1), n_in=510, n_out=100, activation=T.tanh)
# logistic regression layer - note the dimension magic needed to append the category to each input vector
dlayer3 = LogisticRegression(input=dlayer2.output, n_in=100, n_out=2) # 0 is generated, 1 is real
dcost = dlayer3.negative_log_likelihood(zd)
# This approach uses the negative log liklihood that the incorrect answer is selected (p_y_given_x is the softmax of the output of the layer)
gcost = -T.mean(T.log(dlayer3.p_y_given_x)[T.arange(zd.shape[0]), 1-zd]) # using the given for zd
dparams = dlayer3.params + dlayer2.params + dlayer1.params + dlayer0.params
gparams = glayer3.params + glayer2.params + glayer1.params + glayer0.params
dgrads = T.grad(dcost, dparams)
ggrads = T.grad(gcost, gparams)
dopt = rmsprop(dparams)
gopt = rmsprop(gparams)
dupdates = dopt.updates(dparams, dgrads,
learning_rate / np.cast['float32'](batch_size),
momentum)
gupdates = gopt.updates(gparams, ggrads,
learning_rate / np.cast['float32'](batch_size),
momentum)
# functions to train the discriminative model
# important: call gfire first and set current_generated_data_shared and current_generated_categories_shared to the returned values before training
zd_temp = np.zeros(batch_size)
zd_temp[:batch_size//2] = 1
zd_given = theano.shared(np.int32(zd_temp), borrow=True)
index = T.lscalar()
# training occurs on both real and generated training data
dtrain_model = theano.function([index], dcost, updates=dupdates, givens={xd:T.concatenate((training_data_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])),
yd:T.concatenate((training_categories_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])),
zd:zd_given})
# validation is just on the validation data - it sees how many of them it thinks are real
dvalidate_model = theano.function([index], dlayer3.errors(zd), givens={xd:validation_data_shared[index * batch_size: (index+1) * batch_size],
yd:validation_categories_shared[index * batch_size: (index+1) * batch_size],
zd:theano.shared(np.int32(np.ones(batch_size)))})
# testing occurs on real and generated
dtest_model = theano.function([index], dlayer3.errors(zd), givens={xd:T.concatenate((test_data_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])),
yd:T.concatenate((test_categories_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])),
zd:zd_given})
sample = T.matrix('samples')
category = T.ivector('category')
dfire = theano.function([sample, category], dlayer3.p_y_given_x, givens={xd:sample, yd:category})
# category values and random numbers - input to generative network
ginput_categories_shared = theano.shared(np.int32(np.array(range(10)*(batch_size//10))))
ginput_rng_shared = theano.shared(rng.uniform(low=-1,high=1, size=[batch_size,10]))
# function to train the generative model
gtrain_model = theano.function([], gcost, updates=gupdates, givens={xg_category:ginput_categories_shared, xg_seed:ginput_rng_shared, zd:theano.shared(np.int32(np.zeros(batch_size)))})
# function to test/validate the generative model
gtest_model = theano.function([], 1-dlayer3.errors(zd), givens={xg_category:ginput_categories_shared, xg_seed:ginput_rng_shared,zd:theano.shared(np.int32(np.zeros(batch_size)))})
# function to generate new training examples for the discriminative model
gfire = theano.function([], [goutput, xg_category], givens={xg_category:ginput_categories_shared, xg_seed:ginput_rng_shared})
dbest_loss = 1
def generate_new_data():
ginput_rng_shared.set_value(rng.uniform(low=0,high=1, size=[batch_size,10]))
generated_data = np.zeros([NUM_TRAINING_EXAMPLES,1,28,28])
generated_categories = np.zeros_like(current_generated_categories_shared.get_value())
for i in range(NUM_TRAINING_EXAMPLES//batch_size):
generated_data[batch_size * i: batch_size * (i+1)], generated_categories[batch_size * i: batch_size * (i+1)] = gfire()
current_generated_data_shared.set_value(generated_data.reshape([NUM_TRAINING_EXAMPLES,28*28]))
current_generated_categories_shared.set_value(generated_categories)
# training loop
print("Training...")
for epoch in range(1, num_epochs):
print "training epoch: ", epoch
if epoch % 10 == 0:
print "evaluating discriminator..."
# fire the generator before validating
generate_new_data()
# calculate validation loss
dvalidation_loss = [dvalidate_model(i) for i in range(NUM_VALIDATION_EXAMPLES//batch_size)]
mean_dvalidation_loss = np.mean(dvalidation_loss)
print"discrimination error (% of real labeled fake): ", mean_dvalidation_loss * 100, "%"
print "evaluating generator..."
gvalidation_loss = gtest_model()
print"generation quality (% of fake labeled real): ", (1-gvalidation_loss) * 100, "%"
if epoch % 10 == 0:
for i in range(10):
#print "percent certainty that generated image is real: ", dfire(current_generated_data_shared[:batch_size].reshape([batch_size,28*28]),current_generated_categories_shared[:batch_size])[i]
plt.imshow(np.reshape(current_generated_data_shared.get_value()[i], [28,28]), cmap=plt.get_cmap('gray'), interpolation='nearest')
#plt.show()
plt.savefig("output2/epoch" + str(epoch) + "image" + str(current_generated_categories_shared.get_value()[i]) + ".png")
# fire the generator before training
ginput_rng_shared = theano.shared(rng.uniform(low=0,high=1, size=[batch_size,10]))
generate_new_data()
dc = 1
gc = 1
for batch_index in range(NUM_TRAINING_EXAMPLES//batch_size * 2):
if gc*.1 < dc :
dc = dtrain_model(batch_index)
if batch_index % (NUM_TRAINING_EXAMPLES//batch_size) == 0:
print "dcost: ", dc
test = dfire(np.append(training_data_shared.get_value()[0: batch_size//2],
current_generated_data_shared.get_value()[0: batch_size//2], axis=0),
np.append(training_categories_shared.get_value()[0: batch_size//2],
current_generated_categories_shared.get_value()[0:batch_size//2], axis=0))
print "percent certainty that real image is [fake,real]: ", test[0]
print "correct answer: ", zd_given.get_value()[0]
print "percent certainty that generated image is [fake,real]: ", test[batch_size//2]
print "correct answer: ", zd_given.get_value()[batch_size//2]
if dc*.1 < gc :
ginput_rng_shared = theano.shared(rng.uniform(low=0,high=1, size=[batch_size,10]))
gc = gtrain_model()
if batch_index % (NUM_TRAINING_EXAMPLES//batch_size) == 0: print "gcost: ", gc
plt.show()
def loadData():
train_x = []
train_y = []
fileIn = open('testSet_LR.txt')
for line in fileIn.readlines():
lineArr = line.strip().split()
train_x.append([1.0, float(lineArr[0]), float(lineArr[1])])
train_y.append(float(lineArr[2]))
return mat(train_x), mat(train_y).transpose()
## step 1: load data
print "step 1: load data..."
train_x, train_y = loadData()
test_x = train_x; test_y = train_y
## step 2: training...
print "step 2: training..."
opts = {'alpha': 0.01, 'maxIter': 1000, 'optimizeType': 'gradDescent'}
optimalWeights = lr.trainLogRegres(train_x, train_y, opts)
## step 3: testing
print "step 3: testing..."
accuracy = lr.testLogRegres(optimalWeights, test_x, test_y)
## step 4: show the result
print "step 4: show the result..."
print 'The classify accuracy is: %.3f%%' % (accuracy * 100)
lr.showLogRegres(optimalWeights, train_x, train_y)
import csv
import numpy as np
csv_file_object = csv.reader(open('train.csv', 'rb')) #Load in the csv file
header = csv_file_object.next() #Skip the fist line as it is a header
data=[] #Creat a variable called 'data'
for row in csv_file_object: #Skip through each row in the csv file
data.append(row) #adding each row to the data variable
data = np.array(data) #Then convert from a list to an array
#print header
#survived0,pclass1-0,name2,sex3-1,age4-2,sibsp5-3,parch6-4,ticket7,fare8-5,cabin9,embarked10
outputs = np.matrix(data[0:, 0]).T.astype(int)
inputs = data[0:, (1, 3, 4, 5, 6, 8)]
inputs[:, 1] = lr.eapply(lambda x: int(x), data[0::,3] == "male")
def liq(x):
if x == '': return 0
else: return x
inputs = np.matrix(lr.eapply(liq, inputs))
inputs = np.matrix(inputs).T.astype(float)
def clasterize(vector, classCount):
a = list(np.array(vector).reshape(-1))
maxv, minv = max(a), min(a)
h = float(maxv - minv) / classCount
return [ int((x - minv) / h) for x in a]
classCount = 4
clasterized = (2, 3, 5)
import csv
import numpy as np
csv_file_object = csv.reader(open('train.csv', 'rb')) #Load in the csv file
header = csv_file_object.next() #Skip the fist line as it is a header
data=[] #Creat a variable called 'data'
for row in csv_file_object: #Skip through each row in the csv file
data.append(row) #adding each row to the data variable
data = np.array(data) #Then convert from a list to an array
#print header
#survived0,pclass1-0,name2,sex3-1,age4-2,sibsp5-3,parch6-4,ticket7,fare8-5,cabin9,embarked10
outputs = np.matrix(data[0:, 0]).T.astype(int)
inputs = data[0:, (1, 3, 4, 5, 6, 8)]
inputs[:, 1] = lr.eapply(lambda x: int(x), data[0::,3] == "male")
def liq(x):
if x == '': return 0
else: return x
inputs = np.matrix(lr.eapply(liq, inputs))
inputs = np.matrix(inputs).T.astype(float)
w, e = lr.batchGradientDescent(inputs, outputs, np.matrix([[0] * 7]).T, 0.5, -0.01, 100)
print e[0:4], '...'
print e[-5:-1]
"""s = 0.0
for m in range(outputs.shape[0]):
def classify():
[training_data_shared, training_categories_shared,
validation_data_shared, validation_categories_shared,
test_data_shared, test_categories_shared] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES, TEST_DIR, NUM_TEST_EXAMPLES)
# create a nn with two 28**2 5x5 convolutional layer (each node looks at 25 pixels in a square) + convolutional pooling layer, then a regular nn layer
print"building the model..."
rng = np.random.RandomState()
nkerns=[20,50]
batch_size = 100
n_train_batches = NUM_TRAINING_EXAMPLES/batch_size
x = T.matrix('x')
y = T.ivector('y')
index = T.lscalar()
layer0_input = x.reshape((batch_size, 1, 28, 28))
layer0 = ConvPoolingLayer(rng, input=layer0_input, image_shape=(batch_size, 1, 28, 28), filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2,2))
# outputs a convoluted image
layer1 = ConvPoolingLayer(rng, input=layer0.output, image_shape=(batch_size, nkerns[0], 12, 12), filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2,2))
# outputs a doubly convoluted image
# flatten before putting it through the rest of the NN
layer2_input = layer1.output.flatten(2)
layer2 = HiddenLayer(rng, input=layer2_input, n_in=nkerns[1]*4*4, n_out=500, activation=T.tanh) # tanh activation function (sigmoidal)
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
cost = layer3.negative_log_likelihood(y)
test_model = theano.function([index], layer3.errors(y), givens={x:test_data_shared[index * batch_size: (index+1) * batch_size],
y:test_categories_shared[index * batch_size: (index+1) * batch_size]})
validate_model = theano.function([index], layer3.errors(y), givens={x:validation_data_shared[index * batch_size: (index+1) * batch_size],
y:validation_categories_shared[index * batch_size: (index+1) * batch_size]})
# train on the images
print"training..."
params = layer3.params + layer2.params + layer1.params + layer0.params
grads = T.grad(cost, params) # the magic step
updates = [(param_i, param_i - learning_rate * grad_i) for param_i, grad_i in zip(params, grads)]
train_model = theano.function([index], cost, updates=updates, givens={x:training_data_shared[index * batch_size: (index+1) * batch_size],
y:training_categories_shared[index * batch_size: (index+1) * batch_size]})
epoch = 0
best_loss = 1
while (epoch < N_EPOCHS):
epoch +=1
if epoch %5 == 0: print"training epoch: ", epoch
if epoch %5 == 0:
print"validating..."
validation_loss = [validate_model(i) for i in range(NUM_VALIDATION_EXAMPLES/batch_size)]
this_validation_loss = np.mean(validation_loss)
if this_validation_loss < best_loss:
best_loss = this_validation_loss
else:
print "loss is worse now, breaking..."
break
print"loss: ", this_validation_loss * 100, "%"
for minibatch_index in range(n_train_batches):
train_model(minibatch_index)
print"done training! Testing..."
test_losses = [test_model(i) for i in range(NUM_TEST_EXAMPLES/batch_size)]
test_score = np.mean(test_losses)
print"Test score loss: ", test_score * 100, "%"
return self.responses_test
def get_training_predictors(self):
"""
Gets the training predictors.
:return: The training predictors
"""
return self.predictors_training
def get_training_responses(self):
"""
Gets the training responses.
:return: The training responses
"""
return self.responses_training
def get_training_sample_count(self):
"""
Gets the number of training samples.
:return: The number of training samples
"""
return self.training_sample_count
# Create a spam data set object, print a start message, and give a
# demonstration of hand-coded logistic regression using the spam
# data set!
DATA_SET = SpamData()
print("Demonstration of hand-coded logistic gradient descent versus "
"scikit-learn's implementation using the spam data set...")
LogisticRegression.give_demonstration(DATA_SET)
def testThree(self):
z = np.array([[8, 1, 6], [3, 5, 7], [4, 9, 2]])
expected = np.array([[0.9997, 0.7311, 0.9975], [0.9526, 0.9933, 0.9991], [0.9820, 0.9999, 0.8808]])
np.testing.assert_almost_equal(LR.sigmoid(z), expected, decimal=4)
def plot_learning_curve(features_train, labels_train, features_test, labels_test, outputClasses=None, K="linear", C=1,
G=0.01,
method="error", classifier="SVM", a=0, b=1, showFigure=True, saveFigure=False, savePath=None,
nb_epochs=10):
# run for every 10% of training set and compute training error and testing error
step = len(features_train) / 10
train = []
test = []
maj_clas = []
for i in range(0, 10):
print 'iteration : ', i
# train for (i+1)*10 percent of training set
f = features_train[0:((i + 1) * (step))]
l = labels_train[0:((i + 1) * (step))]
assert f.shape[0] == l.shape[0], 'Wrong number of input data! '
if classifier == "SVM":
# train classifier for the specific subset of training set
model = SVM.train(f, l, k=K, c=C, g=G)
# get training error
predictionTrain = SVM.predict(f, model)
# get testing error
predictionTest = SVM.predict(features_test, model)
elif classifier == "LR":
# train classifier for the specific subset of training set
model = LogisticRegression.train(f, l, c=C)
# get training error
predictionTrain = LogisticRegression.predict(f, model)
# get testing error
predictionTest = LogisticRegression.predict(features_test, model)
elif classifier == "CNN":
model = Keras_test.CNN().train(features=f, labels=l,outputClasses=outputClasses,
learning_curves_OR_Cross_Val=True,nb_epoch=nb_epochs)
# get training error
predictionTrain = Keras_test.CNN().predictClasses(features=f, model=model)
# get testing error
predictionTest = Keras_test.CNN().predictClasses(features=features_test, model=model)
# TODO : CNN MINIBATCHES LEARNING CURVES , Implementation : read 10% of data and train cnn.train with all the data
# elif classifier == "CNN_minibatches":
elif classifier == "MLP":
model = Keras_test.MLP().train(features=f, labels=l, outputClasses=outputClasses,
learning_curves_OR_Cross_Val=True, nb_epoch=nb_epochs)
# get training error
predictionTrain = Keras_test.MLP().predict(features=f, model=model, ShowAccuracy=False)
# get testing error
predictionTest = Keras_test.MLP().predict(features=features_test, model=model, ShowAccuracy=False)
elif classifier == "SimpleRNN":
model = Keras_test.RNN().trainRNN(features=f, labels=l, outputClasses=outputClasses, learning_curves=True)
# get training error
predictionTrain = Keras_test.RNN().predict(features=f, model=model, ShowAccuracy=False)
# get testing error
predictionTest = Keras_test.RNN().predict(features=features_test, model=model, ShowAccuracy=False)
elif classifier == "RNN_LSTM":
model = Keras_test.RNN().trainRnnLSTM(features=f, labels=l, outputClasses=outputClasses,
learning_curves=True)
# get training error
predictionTrain = Keras_test.RNN().predict(features=f, model=model, ShowAccuracy=False)
# get testing error
predictionTest = Keras_test.RNN().predict(features=features_test, model=model, ShowAccuracy=False)
# get error for majority classifier
predictionMajority = MajorityClassifier.predictMaj(labels_test)
if method == "error":
train.append(measures.error(l, predictionTrain))
test.append(measures.error(labels_test, predictionTest))
maj_clas.append(measures.error(labels_test, predictionMajority))
elif method == "avgF1":
train.append(measures.avgF1(l, predictionTrain, a, b))
test.append(measures.avgF1(labels_test, predictionTest, a, b))
maj_clas.append(measures.avgF1(labels_test, predictionMajority, a, b))
print test[9]
x = np.arange(len(train)) * 10
plt.plot(x, train, color="blue", linewidth="2.0", label=classifier)
plt.plot(x, test, color="blue", linestyle="dashed", linewidth="2.0")
plt.plot(x, maj_clas, color="red", linewidth="2.0")
plt.ylim(0, 1)
plt.ylabel(method)
plt.xlabel("% of messages")
if method == "error":
plt.legend(loc="upper left")
elif method == "avgF1":
plt.legend(loc="lower left")
if saveFigure:
assert savePath != None, "Give image path to save image"
# with figure i can save it anywhere i want
# fig1 = plt.gcf()
plt.savefig(savePath)
# clear current canvas . if we have show and save together we will have a problem...
plt.clf()
if showFigure:
plt.show()
def buildAdversarialGA():
rng = np.random.RandomState()
print "Loading Data..."
[training_data_shared, training_categories_shared, validation_data_shared, validation_categories_shared, test_data_shared, test_categories_shared] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES, TEST_DIR, NUM_TEST_EXAMPLES)
current_generated_data_shared = theano.shared(np.zeros_like(training_data_shared.get_value()))
current_generated_categories_shared = theano.shared(np.int32(np.zeros_like(training_categories_shared.get_value())))
print "Building the model..."
xd = T.matrix('xd') # input images
yd = T.ivector('yd') # label for the images
yd_onehot = T.extra_ops.to_one_hot(yd, 10)
zd = T.ivector('z') # real or not
dlayer0Input = xd.reshape((batch_size, 1, 28, 28)) # reassemble the image
# convolutional layer
dlayer0 = ConvPoolingLayer(rng, input=dlayer0Input, image_shape=(batch_size, 1, 28, 28), filter_shape=(20, 1, 5, 5), poolsize=(2,2))
# first mlp hidden layer
dlayer1 = HiddenLayer(rng, input=dlayer0.output.flatten(2), n_in=20*9*4*4, n_out=500, activation=T.tanh)
# second mlp hidden layer
dlayer2 = HiddenLayer(rng, input=T.concatenate((dlayer1.output, yd_onehot), axis=1), n_in=510, n_out=100, activation=T.tanh)
# logistic regression layer - note the dimension magic needed to append the category to each input vector
dlayer3 = LogisticRegression(input=dlayer2.output, n_in=100, n_out=2) # 0 is generated, 1 is real
dcost = dlayer3.negative_log_likelihood(zd)
dparams = dlayer3.params + dlayer2.params + dlayer1.params + dlayer0.params
dgrads = T.grad(dcost, dparams)
dopt = rmsprop(dparams)
dupdates = dopt.updates(dparams, dgrads,
learning_rate / np.cast['float32'](batch_size),
momentum)
zd_temp = np.zeros(batch_size)
zd_temp[:batch_size//2] = 1
zd_given = theano.shared(np.int32(zd_temp), borrow=True)
index = T.lscalar()
# training occurs on both real and generated training data
dtrain_model = theano.function([index], dcost, updates=dupdates, givens={xd:T.concatenate((training_data_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])),
yd:T.concatenate((training_categories_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])),
zd:zd_given})
# validation is just on the validation data - it sees how many of them it thinks are real
dvalidate_model = theano.function([index], dlayer3.errors(zd), givens={xd:validation_data_shared[index * batch_size: (index+1) * batch_size],
yd:validation_categories_shared[index * batch_size: (index+1) * batch_size],
zd:theano.shared(np.int32(np.ones(batch_size)))})
# testing occurs on real and generated
dtest_model = theano.function([index], dlayer3.errors(zd), givens={xd:T.concatenate((test_data_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])),
yd:T.concatenate((test_categories_shared[index * batch_size//2: (index+1) * batch_size//2],
current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])),
zd:zd_given})
sample = T.matrix('samples')
category = T.ivector('category')
real = T.ivector('real?')
# fire the discriminator on input samples, what category they are trying to be and whether they're real or generated (1 == real)
dfire = theano.function([sample, category], dlayer3.p_y_given_x, givens={xd:sample, yd:category})
# fitness function needs to accomidate a list of genes - currently it only accepts batch_size at once, so it loops
def fitness_function(a):
retval = np.array([])
for i in range(len(a)//batch_size):
retval = np.append(retval, dfire(np.packbits(a, axis=1)[i*batch_size: (i+1) * batch_size, :], [digit_to_generate]*batch_size)[:,1])
return retval
current_generated_categories_shared.set_value([digit_to_generate]*NUM_TRAINING_EXAMPLES)
gmodel = pvga(fitness_function, 28*28*8, rng, num_samples_per_gen=2000)
mutation_rate = .05
print "Training..."
for epoch in range(1, num_epochs):
print "Epoch: ", epoch
if epoch %10 == 0:
#print "Validating..."
#error = np.array([])
#for batch_index in range(NUM_VALIDATION_EXAMPLES//batch_size *2):
#error = np.append(error, dvalidate_model(batch_index))
#print "D error: ", np.mean(error) * 100, "%"
plt.imshow(np.reshape(current_generated_data_shared.get_value()[0],[28,28]), cmap=plt.get_cmap('gray'), interpolation='nearest')
plt.draw()
data = np.packbits(gmodel.sample_pop(NUM_TRAINING_EXAMPLES),axis=1)
current_generated_data_shared.set_value(data.reshape((NUM_TRAINING_EXAMPLES, 28*28)))
print "Training D:"
loss = np.array([])
for batch_index in range(NUM_TRAINING_EXAMPLES//batch_size * 2):
loss = np.append(loss, dtrain_model(batch_index))
print "Average loss: ", np.mean(loss)
print "Training G:"
gmodel.create_next_generation(mutation_rate=mutation_rate)
prev_max = gmodel.max_fitness
for i in range(epoch * 50):
#mutation_rate/=2
print "Max fitness: ", gmodel.max_fitness
gmodel.create_next_generation(mutation_rate=mutation_rate)
prev_max = gmodel.max_fitness
plt.show()
def testOne(self):
np.testing.assert_almost_equal(LR.sigmoid(1200000), 1)
np.testing.assert_almost_equal(LR.sigmoid(-250), 0)
np.testing.assert_almost_equal(LR.sigmoid(0), 0.5)
yield (inputs[p])[idx:idx + batchsize], (targets[p])[idx:idx +
batchsize]
print("Loading the data")
X_train, y_train, X_test, y_test = mnist.load_dataset(args['nbtrain'],
args['nbtest'], False)
############################################################################
# Model building
print("Model architecture : ")
filename_prefix = ""
if ('logreg' in args):
print(" Logistic regression, Input -> Softmax")
model = LogisticRegression.Model(28, 28, 1, 10)
filename_prefix += "logreg_"
elif 'mlp' in args:
print(" Multilayer perceptron with : ")
filename_prefix += "mlp_"
sys.exit()
elif 'cnn' in args:
print(" Convolutional neural network with :")
filename_prefix += "cnn_"
sys.exit()
elif 'vgg' in args:
print(" Oxford Visual Geometry Group with :")
filename_prefix += "vgg_"
sys.exit()
else:
sys.exit()
import numpy as np from random import seed, gauss import LogisticRegression as lr i = np.matrix([ [1,1], [1,2], [1,3], [2,1], [2,2], [2,3], [3,1], [3,2], [3,3] ]).T o = np.matrix([[0, 0, 1, 0, 1, 1, 1, 1, 1]]).T w = lr.batchGradientDescent( lr.LogisticDerivative, i, o, np.matrix([[0], [0], [0]]), 0.5, -0.5, 70) print w for i in xrange(3): for v in xrange(3): print i, v, lr.calcLogReg(w, np.matrix([i+1,v+1]).T, 0.5) print lr.calcLogReg(w, np.matrix([3,0.85352157]).T, 0.5)
test_output_labels[:, :, i] = datum.label()
numberOfData = train_input_data.shape[2]
normalizeInputSum = np.sum(train_input_data, axis=2)
normalizeInputSum = np.tile(normalizeInputSum, [numberOfData, 1, 1]).transpose(1, 2, 0)
train_input_data = np.divide(np.power(train_input_data, 2), normalizeInputSum)
numberOfData = test_input_data.shape[2]
normalizeInputSum = np.sum(test_input_data, axis=2)
normalizeInputSum = np.tile(normalizeInputSum, [numberOfData, 1, 1]).transpose(1, 2, 0)
test_input_data = np.divide(np.power(test_input_data, 2), normalizeInputSum)
# V, W = NeuralNetwork.trainNetwork(train_input_data[:, :, 0:100], train_output_labels[0:100], 100, 8, 0.0000001)
lrV = LogisticRegression.trainNetwork(train_input_data, train_output_labels, 100, 0.01);
lrV2 = LogisticRegression2.trainNetwork(train_input_data[0:100], train_output_labels[0:100], 0.001)
# np.save('data/V-3', V)
# np.save('data/W-3', W)
np.save('data/lrV', lrV)
np.save('data/lrV2', lrV2)
# V = np.load('data/V-2.npy')
# W = np.load('data/W-2.npy')
# error, labels = NeuralNetwork.testNetwork(V, W, test_input_data[:, :, 4000:6000], test_output_labels[:, :, 4000:6000], 8)
# test_labels = np.argmax(test_output_labels[:, :, 4000:6000].reshape([2, 2000]), axis=0)
# print np.sum(test_labels == labels)
import numpy as np import matplotlib.pyplot as plt from sklearn import datasets iris = datasets.load_iris() X = iris.data y = iris.target print(X,y) X = X[y<2,:2] y = y[y<2] X.shape (100, 2) y.shape (100,) plt.scatter(X[y==0,0], X[y==0,1], color="red") plt.scatter(X[y==1,0], X[y==1,1], color="blue") plt.show() from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_test = train_test_split(X, y) import LogisticRegression log_reg = LogisticRegression() log_reg.fit(X_train, y_train) LogisticRegression() log_reg.score(X_test, y_test) log_reg.predict_proba(X_test) log_reg.predict(X_test)
class DBN(object):
"""Deep Belief Network
A deep belief network is obtained by stacking several RBMs on top of each
other. The hidden layer of the RBM at layer `i` becomes the input of the
RBM at layer `i+1`. The first layer RBM gets as input the input of the
network, and the hidden layer of the last RBM represents the output. When
used for classification, the DBN is treated as a MLP, by adding a logistic
regression layer on top.
"""
def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500], n_outs=10):
"""This class is made to support a variable number of layers.
:type numpy_rng: numpy.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the input to the DBN
:type hidden_layers_sizes: list of ints
:param hidden_layers_sizes: intermediate layers size, must contain
at least one value
:type n_outs: int
:param n_outs: dimension of the output of the network
"""
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector
# of [int] labels
# end-snippet-1
# The DBN is an MLP, for which all weights of intermediate
# layers are shared with a different RBM. We will first
# construct the DBN as a deep multilayer perceptron, and when
# constructing each sigmoidal layer we also construct an RBM
# that shares weights with that layer. During pretraining we
# will train these RBMs (which will lead to chainging the
# weights of the MLP as well) During finetuning we will finish
# training the DBN by doing stochastic gradient descent on the
# MLP.
for i in range(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden
# units of the layer below or the input size if we are on
# the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the
# hidden layer below or the input of the DBN if you are on
# the first layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question... but we are
# going to only declare that the parameters of the
# sigmoid_layers are parameters of the DBN. The visible
# biases in the RBM are parameters of those RBMs, but not
# of the DBN.
self.params.extend(sigmoid_layer.params)
# Construct an RBM that shared weights with this layer
rbm_layer = RBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
# compute the cost for second phase of training, defined as the
# negative log likelihood of the logistic regression (output) layer
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
def pretraining_functions(self, train_set_x, batch_size, k):
'''Generates a list of functions, for performing one step of
gradient descent at a given layer. The function will require
as input the minibatch index, and to train an RBM you just
need to iterate, calling the corresponding function on all
minibatch indexes.
:type train_set_x: theano.tensor.TensorType
:param train_set_x: Shared var. that contains all datapoints used
for training the RBM
:type batch_size: int
:param batch_size: size of a [mini]batch
:param k: number of Gibbs steps to do in CD-k / PCD-k
'''
# index to a [mini]batch
index = T.lscalar('index') # index to a minibatch
learning_rate = T.scalar('lr') # learning rate to use
# number of batches
n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# begining of a batch, given `index`
batch_begin = index * batch_size
# ending of a batch given `index`
batch_end = batch_begin + batch_size
pretrain_fns = []
for rbm in self.rbm_layers:
# get the cost and the updates list
# using CD-k here (persisent=None) for training each RBM.
# TODO: change cost function to reconstruction error
cost, updates = rbm.get_cost_updates(learning_rate,
persistent=None, k=k)
# compile the theano function
fn = theano.function(
inputs=[index,
theano.In(learning_rate, value=0.1)],
outputs=cost,
updates=updates,
givens={
self.x: train_set_x[batch_begin:batch_end]
}
)
# append `fn` to the list of functions
pretrain_fns.append(fn)
return pretrain_fns
def build_finetune_functions(self, datasets, batch_size, learning_rate):
'''Generates a function `train` that implements one step of
finetuning, a function `validate` that computes the error on a
batch from the validation set, and a function `test` that
computes the error on a batch from the testing set
:type datasets: list of pairs of theano.tensor.TensorType
:param datasets: It is a list that contain all the datasets;
the has to contain three pairs, `train`,
`valid`, `test` in this order, where each pair
is formed of two Theano variables, one for the
datapoints, the other for the labels
:type batch_size: int
:param batch_size: size of a minibatch
:type learning_rate: float
:param learning_rate: learning rate used during finetune stage
'''
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of minibatches for training, validation and testing
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_valid_batches /= batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_test_batches /= batch_size
index = T.lscalar('index') # index to a [mini]batch
# compute the gradients with respect to the model parameters
gparams = T.grad(self.finetune_cost, self.params)
# compute list of fine-tuning updates
updates = []
for param, gparam in zip(self.params, gparams):
updates.append((param, param - gparam * learning_rate))
train_fn = theano.function(
inputs=[index],
outputs=self.finetune_cost,
updates=updates,
givens={
self.x: train_set_x[
index * batch_size: (index + 1) * batch_size
],
self.y: train_set_y[
index * batch_size: (index + 1) * batch_size
]
}
)
test_score_i = theano.function(
[index],
self.errors,
givens={
self.x: test_set_x[
index * batch_size: (index + 1) * batch_size
],
self.y: test_set_y[
index * batch_size: (index + 1) * batch_size
]
}
)
valid_score_i = theano.function(
[index],
self.errors,
givens={
self.x: valid_set_x[
index * batch_size: (index + 1) * batch_size
],
self.y: valid_set_y[
index * batch_size: (index + 1) * batch_size
]
}
)
# Create a function that scans the entire validation set
def valid_score():
return [valid_score_i(i) for i in range(n_valid_batches)]
# Create a function that scans the entire test set
def test_score():
return [test_score_i(i) for i in range(n_test_batches)]
return train_fn, valid_score, test_score
def sigmoid_activate(self, Xtest, W, b):
# code and compute
sigmoid_input = Xtest
sigmoid_output = numpy.tanh(numpy.dot(sigmoid_input, W.get_value(borrow=True)) + b.get_value(borrow=True))
return sigmoid_output
def softmax_activate(self, Xtest, logLayer):
# code and compute
softmax_input = Xtest
v = numpy.exp( numpy.dot(softmax_input, logLayer.W.get_value(borrow=True)) + logLayer.b.get_value(borrow=True))
softmax_output = v/numpy.sum(v)
return softmax_output
def predict_functions(self, Xtest):
''' Given a set_x of examples produce a vector y' of predictions by the DBN.
'''
tmp = Xtest
for L in self.sigmoid_layers:
tmp = self.sigmoid_activate( tmp, L.W, L.b )
# finalize with log layer
tmp = self.softmax_activate( tmp, self.logLayer )
return tmp
def main():
np.random.seed(seed=1)
products = pd.read_csv('../Datafiles/amazon_baby.csv')
review_without_punctuation = products['review'].apply(
LR.remove_punctuation)
products['word_count'] = review_without_punctuation.apply(LR.countWords)
products['clear_words'] = review_without_punctuation
# print 5 items in the list
print review_without_punctuation.head()
# remove all rating 3 , they are considered as neutral
products = products[products['rating'] != 3]
products['sentiment'] = products['rating'].apply(lambda rating: +1
if rating > 3 else -1)
mask = np.random.rand(len(products)) < 0.8
train = products[mask]
test = products[~mask]
print "Training data size", len(train), "Testing data size", len(test)
train_reviews = np.array(train['clear_words'])
train_labels = np.array(train['sentiment'])
vect = CountVectorizer().fit(train_reviews)
trained_vectorized = vect.transform(train_reviews)
sentiment_model = LogisticRegression()
sentiment_model.fit(trained_vectorized, train_labels)
print sentiment_model.coef_
print np.sum(sum(sentiment_model.coef_ >= 0))
sample_test_data = test[10:13]
print sample_test_data
# 1, 1, -1 for sentiment
sample_test = vect.transform(sample_test_data['clear_words'])
# first test
print sentiment_model.predict(sample_test)
# result is 1, 1, -1
scores = sentiment_model.decision_function(sample_test)
probability = LR.compute_probability(scores)
print probability
# start to use test data
test_vect = vect.transform(test['clear_words'])
test_pred = sentiment_model.predict(test_vect)
test_scores = sentiment_model.decision_function(test_vect)
test_prob = LR.compute_probability(test_scores)
test['predictions'] = test_pred
test.sort('predictions', ascending=False)
print test.head(20)
def __init__(self, numpy_rng, theano_rng=None, n_ins=784,
hidden_layers_sizes=[500, 500], n_outs=10):
"""This class is made to support a variable number of layers.
:type numpy_rng: numpy.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the input to the DBN
:type hidden_layers_sizes: list of ints
:param hidden_layers_sizes: intermediate layers size, must contain
at least one value
:type n_outs: int
:param n_outs: dimension of the output of the network
"""
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector
# of [int] labels
# end-snippet-1
# The DBN is an MLP, for which all weights of intermediate
# layers are shared with a different RBM. We will first
# construct the DBN as a deep multilayer perceptron, and when
# constructing each sigmoidal layer we also construct an RBM
# that shares weights with that layer. During pretraining we
# will train these RBMs (which will lead to chainging the
# weights of the MLP as well) During finetuning we will finish
# training the DBN by doing stochastic gradient descent on the
# MLP.
for i in range(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden
# units of the layer below or the input size if we are on
# the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the
# hidden layer below or the input of the DBN if you are on
# the first layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question... but we are
# going to only declare that the parameters of the
# sigmoid_layers are parameters of the DBN. The visible
# biases in the RBM are parameters of those RBMs, but not
# of the DBN.
self.params.extend(sigmoid_layer.params)
# Construct an RBM that shared weights with this layer
rbm_layer = RBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
# compute the cost for second phase of training, defined as the
# negative log likelihood of the logistic regression (output) layer
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
import matplotlib.pyplot as plt
import TwoDimensionPlot
##################################模型1########################################
#导入数据集
pima = pd.read_csv('pima-indians-diabetes.csv')
pd0 = pima[pima.columns.tolist()[0:len(pima.columns) - 1]]
pd1 = pima['label']
#归一化
pd0 = pd.DataFrame(preprocessing.scale(pd0))
pima = pd.concat([pd0, pd1], axis=1)
pima_train, pima_test = train_test_split(pima, test_size=0.3)
#定义模型：迭代1000次，学习率默认为0.0001，训练方式随机梯度下降
model_lr1 = LogisticRegression.LogisticRegression(n_iter=1000, gd='sgd')
#训练模型
model_lr1.fit(pima_train)
print("模型1训练集精确率：", model_lr1.accuracy(pima_train))
print("模型1测试集精确率：", model_lr1.accuracy(pima_test))
print("模型1权重为：", model_lr1.weights[0])
#导入数据集
data = pd.read_csv('test.csv')
#划分数据集
data_train, data_test = train_test_split(data, test_size=0.3)
#############################模型2#############################################
##定义模型：迭代10000次，学习率为0.01，训练方式默认批量梯度下降，使用L2正则化
model_lr2 = LogisticRegression.LogisticRegression(eta=0.01,
n_iter=10000,
def DisplayDigit(sample, label):
sample = sample.reshape(settings.WIDTH, settings.HEIGHT)
[fig, axs] = plt.subplots(1,1)
axs.imshow(sample, cmap='Greys_r')
print label
############### main function #################
if __name__ == '__main__':
SetupPath()
from settings import *
from ReadDataSet import *
from LogisticRegression import *
import settings
settings.init()
dataset = ReadDataSet()
dataset.Read(GetCurrentPath() + '/Data/mnist.pkl')
[train_data, train_labels] = dataset.getTrainData()
DisplayDigit(train_data[8], train_labels[8])
LR = LogisticRegression(train_data, -1, train_labels)
Wt = LR.train(settings.ETA, settings.EPOCH)
import cPickle
with open(GetCurrentPath() + '/Models/weights.pkl', 'wb') as fp:
cPickle.dump(Wt, fp)
# 10-701 Machine Learning, Spring 2011: Homework 3
# Solution code for question 5 of Section 2
import numpy as np
import LogisticRegression
X_tr = np.loadtxt(open("Data/tr_X.txt", "rb"), delimiter=",")
y_tr = np.loadtxt(open("Data/tr_y.txt", "rb"), delimiter=",")
X_te = np.loadtxt(open("Data/te_X.txt", "rb"), delimiter=",")
y_te = np.loadtxt(open("Data/te_y.txt", "rb"), delimiter=",")
'''
# Let's see the digit represented by this vector
plt.imshow(X_tr[1000].reshape((16,16)))
plt.gray()
plt.show()
'''
model = LogisticRegression.LogisticRegression()
# Pass different values of lambda for solution of 5(b)
model.train_model(X_tr, y_tr, X_te, y_te, lamda=1000)
def testTwo(self):
z = np.array([4, 5, 6])
expected = np.array([0.9820, 0.9933, 0.9975])
np.testing.assert_almost_equal(LR.sigmoid(z), expected, decimal=4)
