def main_process(train_path, test_path, selected_features): # 1)get training set data that used to train model train_data, train_label, m = log.get_train_data_and_label(train_path, selected_features) print("获取到的数据:", train_data, "----格式为:", train_data.shape) # 2)initialise the parameter of logistic regression model theta = log.initialise_logistic_regression_params(train_data) # train_data is X print("初始化的权值格式", theta.shape) # 3)train the model to get the parameter theta (theta is a weight matrix) trained_theta = log.batch_gradient_descent(train_data, train_label, theta, alpha, max_iterations) print("训练后得到的权值为", trained_theta, "--格式为:", trained_theta.shape) # 4)get the test set data that used for prediction test_data, test_id = process_test_set(test_path, selected_features) print("测试数据为:", test_data, "---格式为:", test_data.shape) # 5)do prediction prediction_result_y = log.predict(test_data, trained_theta) # theta_default训练后的theta print("预测的结果为:", np.mat(prediction_result_y), "格式:", prediction_result_y.shape) # 6)transform prediction results into submission format result_list = [] for i in range(len(prediction_result_y)): # print yPre[i, 0] if prediction_result_y[i, 0] >= 0.5: result_list.append(1) elif prediction_result_y[i, 0] < 0.5: result_list.append(0) y_array = np.array(result_list) return y_array, test_id
def plot_recall_precision(length, features_train, labels_train, features_test, labels_test): # threshold=[0.1 ,0.2 ,0.3 ,0.4,0.5,0.6,0.7,0.8,0.9] threshold = [x / 1000.0 for x in range(0, 1001, 1)] step = length / 3 colors = ['b', 'r', 'g'] for i in range(0, 3): # ((i+1)*(step)) percent of train data f = features_train[0:((i + 1) * (step))] l = labels_train[0:((i + 1) * (step))] # train classifier for the specific subset of training set model = LogisticRegression.train(f, l) # recall-precision for every threshold value recall = [] precision = [] for t in threshold: prediction = LogisticRegression.predict(features_test, model, t) recall.append(measures.recall(labels_test, prediction, 0)) precision.append(measures.precision(labels_test, prediction, 0)) plt.plot(recall, precision, linewidth="2.0", label=str((i + 1) * 33) + "% of train data", color=colors[i]) plt.xlim(0, 1) plt.ylim(0, 1) plt.xlabel('Recall') plt.ylabel('Precision') plt.title('Negative tweets') plt.legend() plt.show()
def testLogisticRegression(self): dataset = datasets.load_breast_cancer() data = dataset.data target = dataset.target X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2) scaler = StandardScaler() scaler.fit(X_train) X_train_scaled = scaler.transform(X_train) X_test_scaled = scaler.transform(X_test) clf = linear_model.LogisticRegressionCV() clf.fit(X_train_scaled, y_train) pred_skl = clf.predict(X_test_scaled) model = LogisticRegression() model.fit(X_train_scaled, y_train) pred_model = model.predict(X_test_scaled) accuracy_test_skl = accuracy_score(pred_skl, y_test) accuracy_test_model = accuracy_score(pred_model, y_test) accuracy_model_skl = accuracy_score(pred_skl, pred_model) print('scikit accuracy:', accuracy_test_skl) print('model accuracy:', accuracy_test_model) print('scikit vs model accuracy: ', accuracy_model_skl)
def trainData3(folder, gradDecent): datas, labels = LR.loadDataSet(folder, "data3.txt"); datasMap = []; degree = 6; m = np.shape(datas)[0]; for k in range(m): tmp = [1]; for i in range(degree): for j in range(i + 2): tmp.append(pow(datas[k][0], i + 1 - j) * pow(datas[k][1], j)); datasMap.append(tmp); datas = np.mat(datasMap); alpha = 1; lamb = 1; # itemNum = 100; # batch itemNum = 500; thetas = gradDecent(datas, labels, alpha, lamb, itemNum); print "thetas: ", thetas; thetas = np.mat(thetas); m = np.shape(datas)[0]; error = 0; for i in range(m): pred = LR.classfy(thetas, np.mat(datas[i])); if pred != labels[i]: error = error + 1; errorRate = float(error) * 100 / m; print "The error rate of the test is %f percents" % errorRate thetas = np.array(thetas)[0]; Plot.plot(folder, "data3.txt", thetas);
def call_logistic(num_iterations=2000, alpha=0.5, print_cost=False): ''' Desc : perform logistic regression on given dataset Input: 1. num_iterations -> num of times to run gradient descent 2. alpha -> learning rate 3. print_cost -> prints the cost function every 100 iterations if set to "True" Output: Predictions ''' ## Load data for pre-processing train_set_x_orig, Y_train, test_set_x_orig, Y_test, classes = lr.load_dataset( ) # flattening the train and test data, converting from 3D to 1D train_set_x_flatten = train_set_x_orig.reshape(train_set_x_orig.shape[0], -1).T test_set_x_flatten = test_set_x_orig.reshape(test_set_x_orig.shape[0], -1).T # standerdizing the data X_train = train_set_x_flatten / 255 X_test = test_set_x_flatten / 255 ## building final model w, b = lr.initialize_parameters(X_train.shape[0]) # fitting best values of w and b on data parameters, grads, costs = lr.fit(w, b, X_train, Y_train, num_iterations, alpha, print_cost) w = parameters["w"] b = parameters["b"] Y_prediction_train = lr.predict(w, b, X_train) Y_prediction_test = lr.predict(w, b, X_test) print("train accuracy: {} %".format( 100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100)) print("test accuracy: {} %".format( 100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100)) d = { "costs": costs, "Y_prediction_test": Y_prediction_test, "Y_prediction_train": Y_prediction_train, "w": w, "b": b, "alpha": alpha, "num_iterations": num_iterations } return d
def PlotXference_AVG(): global EyeData global Events #Change these into arrays and prelocate size inference = [] noference = [] for idx in range(0, len(Events)): # inference.append(FindSlices(EyeData[idx], Events[idx], 'Inference', trialTypes)) # noference.append(FindSlices(EyeData[idx], Events[idx], 'Noference', trialTypes)) inference.append( FindSlices(EyeData[idx], Events[idx], 'Inference', 'typeB', 1)) noference.append( FindSlices(EyeData[idx], Events[idx], 'Noference', 'typeA', 1)) fig = plt.figure() fig.suptitle('Gaze X position') ax = fig.add_subplot(121) ax2 = fig.add_subplot(122) for trial in inference: ax.plot(trial) ax.set_ylim(0, 2000) ax.set_ylabel('X coordinate of gaze position') ax.set_xlabel('Inference trials \n x time course in ms') for trial in noference: ax2.plot(trial) ax2.set_ylim(0, 2000) ax2.set_xlabel('No inference trials \n x time course in ms') ticks = ax.get_xticks() * 16 ax.set_xticklabels(ticks.astype(int)) ax2.set_xticklabels(ticks.astype(int)) inf_cat = [1 for i in range(1, len(inference) + 1)] nof_cat = [0 for i in range(1, len(noference) + 1)] known_cat = np.hstack((np.array(inf_cat), np.array(nof_cat))) ferences = np.vstack((inference, noference)) PlotAverage_X(np.array(inference), np.array(noference)) # components = PCA.myPCA(ferences, known_cat) components = components * 1000 LOG_REG.logReg(known_cat, components) #components_tmp = components *1000 np.savetxt("eda_pcaResults.csv", np.hstack((known_cat.reshape(len(known_cat), 1), components)), delimiter=",")
def colicTest(): ''' 对数据格式化处理 ''' frTrain = open('horseColicTraining.txt') frTest = open('horseColicTest.txt') trainingSet = [] trainingLabels = [] for line in frTrain.readlines(): currLine = line.strip().split('\t') lineArr = [] for i in range(21): lineArr.append(float(currLine[i])) trainingSet.append(lineArr) trainingLabels.append(float(currLine[21])) trainWeights = LR.stocGradAscent1(array(trainingSet), trainingLabels, 500) errorCount = 0 numTestVec = 0.0 #读取测试集数据,计算分类错误的样本条数和最终的错误率 for line in frTest.readlines(): numTestVec += 1.0 currLine = line.strip().split('\t') lineArr = [] for i in range(21): lineArr.append(float(currLine[i])) if int(classifyVector(array(lineArr), trainWeights)) != int( currLine[21]): errorCount += 1 errorRate = (float(errorCount) / numTestVec) print("the error rate is: %f" % errorRate) return errorRate
def main(): X, y = generateData() start = timer() test_x, test_y = generateData(1992) clf = nn.multilayerperceptron(layers=[ nn.layer("tanh", 3), nn.layer("tanh", 5), nn.layer("tanh", 5), nn.layer("softmax", 2) ], opt_function='momentum', drop_out=1) clf = lr.logisticregression() #clf = linear_model.LogisticRegression(solver="sag") yt = pd.get_dummies(y, prefix='class').values tyt = pd.get_dummies(test_y, prefix='class').values clf.fit(test_x, test_y) print(clf.coef_) timeusage = timer() - start print("%f seconds" % timeusage) y_ = clf.predict(X) count = 0 for i in range(len(y)): if (y[i] == y_[i]): count += 1 print(count / len(y)) # Plot the decision boundary plot_decision_boundary(lambda x: clf.predict(x), X, y) plt.title("Multilayer Perceptron") plt.show()
def colicTest(): frTrain = open('horseColicTraining.txt'); frTest = open('horseColicTest.txt') trainingSet = []; trainingLabels = [] for line in frTrain.readlines(): currLine = line.strip().split('\t') lineArr =[] for i in range(21): lineArr.append(float(currLine[i])) trainingSet.append(lineArr) trainingLabels.append(float(currLine[21])) '''计算回归系数向量这里可以自由设定迭代的次数,例如在训练集 上使用500次迭代,实验结果表明这比默认迭代150次的效果更好。''' trainWeights = LogisticRegression.stocGranAscentImprove(array(trainingSet), trainingLabels, 1000) '''在系数计算完成之后,导人测试集并计算分类错误率''' errorCount = 0; numTestVec = 0.0 for line in frTest.readlines(): numTestVec += 1.0 currLine = line.strip().split('\t') lineArr =[] for i in range(21): lineArr.append(float(currLine[i])) if int(classifyVector(array(lineArr), trainWeights))!= int(currLine[21]): errorCount += 1 errorRate = (float(errorCount)/numTestVec) print "the error rate of this test is: %f" % errorRate return errorRate
def trainDataset(X, Y, val_size, seed): X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed) # Spot Check Algorithms models = [] models.append(('LR', LogisticRegression())) models.append(('LDA', LinearDiscriminantAnalysis())) models.append(('KNN', KNeighborsClassifier())) models.append(('CART', DecisionTreeClassifier())) models.append(('NB', GaussianNB())) models.append(('SVM', SVC())) # evaluate each model in turn results = [] names = [] print('\n') for name, model in models: kfold = model_selection.KFold(n_splits=10, random_state=seed) cv_results = model_selection.cross_val_score(model, X_train, Y_train.values.ravel(), cv=kfold, scoring='accuracy') results.append(cv_results) names.append(name) msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std()) print(msg) print('\n') return results;
def C_comparison(length, features_train, labels_train, features_test, labels_test): C = [0.001, 0.05, 0.1, 0.3, 0.5, 0.8, 1, 10, 100, 350, 500, 1000, 3500, 5000, 10000, 50000, 100000] scores = [] for c in C: model = LogisticRegression.train(features_train, labels_train, c) prediction = LogisticRegression.predict(features_test, model) scores.append((measures.avgF1(labels_test, prediction, 0, 1))) plt.plot(C, scores, color="blue", linewidth="2.0") plt.xticks(C) plt.ylabel("F1") plt.xlabel("C") plt.show()
def test_analyze_logreg(self): X, y = gen_logistic_data() solver = logReg.LogisticRegressionSolver() with CapturedStdout(): analyzerResults = analyze(solver, X, y, optimizationParams={ "nnTopology": "", "Lambda": 1, "functions": None }, iterations=40, bins=3, tries=4, sample_iterations=4) npt.assert_equal( analyzerResults.sampleCountAnalyzis.sampleCount, [4, 6, 8, 14, 20, 38, 56, 112, 168, 334, 500, 1000, 1500]) npt.assert_almost_equal( analyzerResults.sampleCountAnalyzis.errorTrain, [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], 2) npt.assert_almost_equal(analyzerResults.sampleCountAnalyzis.errorCV, [ 0.0646667, 0.104, 0.1846667, 0.0126667, 0.0166667, 0.0053333, 0.0006667, 0.0006667, 0., 0.0006667, 0., 0., 0. ]) npt.assert_equal(analyzerResults.iterationCountAnalyzis.iterationCount, [2, 4, 6, 10, 14, 27, 40]) npt.assert_almost_equal( analyzerResults.iterationCountAnalyzis.errorTrain, [0., 0.0005, 0., 0., 0., 0., 0.], 5) npt.assert_almost_equal(analyzerResults.iterationCountAnalyzis.errorCV, [0, 0, 0, 0, 0, 0, 0], 5)
def main(): X, y = generateData() start = timer() test_x,test_y = generateData(1992) clf = nn.multilayerperceptron(layers=[ nn.layer("tanh",3), nn.layer("tanh",5), nn.layer("tanh",5), nn.layer("softmax",2) ],opt_function='momentum',drop_out=1) clf = lr.logisticregression() #clf = linear_model.LogisticRegression(solver="sag") yt = pd.get_dummies(y,prefix='class').values tyt = pd.get_dummies(test_y,prefix='class').values clf.fit(test_x, test_y) print(clf.coef_ ) timeusage = timer() - start print("%f seconds"%timeusage) y_ = clf.predict(X) count = 0; for i in range(len(y)): if (y[i] == y_[i]): count += 1 print(count / len(y)) # Plot the decision boundary plot_decision_boundary(lambda x: clf.predict(x),X,y) plt.title("Multilayer Perceptron") plt.show()
def plotFeaturesF1(features_train, labels_train, features_test, labels_test): x = list(np.arange(len(features_train[0]))) # x = list(np.arange(5)) y = [] for i in range(0, len(features_train[0])): f_train = features_train[:, i] f_test = features_test[:, i] f_train = f_train.reshape(f_train.shape[0], 1) f_test = f_test.reshape(f_test.shape[0], 1) model = LogisticRegression.train(f_train, labels_train) prediction = LogisticRegression.predict(f_test, model) y.append(measures.avgF1(labels_test, prediction, 0, 1)) plt.plot(x, y, color="blue", linewidth="2.0") plt.ylabel("F1") plt.xlabel("# of Feature") plt.xticks(x) plt.show()
def __init__(self,n_in,n_hidden_layers,n_hidden_units,n_out): self.n_hidden_layers=n_hidden_layers; self.n_hidden_units=n_hidden_units; self.n_in=n_in; self.n_out=n_out; if n_hidden_layers==0: n_hidden_units=n_in; self.hiddenLayer = [HiddenLayer(n_in=n_in, n_out=n_hidden_units,activation=sigmoid_stable) for i in range(n_hidden_layers)]; self.logRegressionLayer = LogisticRegression.LogisticRegression(n_hidden_units,n_out);
def __init__(self, csv, variables_indeces, classification_values): # ---------- ESTRAPOLA DATI ---------- self.values = csv.giveme_values_indeces(variables_indeces, 0) self.filterY(classification_values[0]) # ---------- MODELLO ---------- self.classification_values = classification_values self.model = LG.LogisticRegression(self.values) # REGRESSIONE self.thetas = None
def testOne(self): X = np.array([[8, 1, 6], [3, 5, 7], [4, 9, 2], [8, 1, 6], [3, 5, 7], [4, 9, 2]]) y = np.array([[1], [0], [1], [0], [1], [0]]) theta = np.array([[0], [1], [0]]) expectedJ = 2.6067 expectedTheta = np.array([[1.7760], [2.3988], [1.9464]]) [actualJ, actualTheta] = LR.costFunction(X, y, theta) np.testing.assert_almost_equal(actualJ, expectedJ, decimal=4) np.testing.assert_almost_equal(actualTheta, expectedTheta, decimal=4)
def PlotXference_AVG(): global EyeData global Events # Change these into arrays and prelocate size inference = [] noference = [] for idx in range(0, len(Events)): # inference.append(FindSlices(EyeData[idx], Events[idx], 'Inference', trialTypes)) # noference.append(FindSlices(EyeData[idx], Events[idx], 'Noference', trialTypes)) inference.append(FindSlices(EyeData[idx], Events[idx], "Inference", "typeB", 1)) noference.append(FindSlices(EyeData[idx], Events[idx], "Noference", "typeA", 1)) fig = plt.figure() fig.suptitle("Gaze X position") ax = fig.add_subplot(121) ax2 = fig.add_subplot(122) for trial in inference: ax.plot(trial) ax.set_ylim(0, 2000) ax.set_ylabel("X coordinate of gaze position") ax.set_xlabel("Inference trials \n x time course in ms") for trial in noference: ax2.plot(trial) ax2.set_ylim(0, 2000) ax2.set_xlabel("No inference trials \n x time course in ms") ticks = ax.get_xticks() * 16 ax.set_xticklabels(ticks.astype(int)) ax2.set_xticklabels(ticks.astype(int)) inf_cat = [1 for i in range(1, len(inference) + 1)] nof_cat = [0 for i in range(1, len(noference) + 1)] known_cat = np.hstack((np.array(inf_cat), np.array(nof_cat))) ferences = np.vstack((inference, noference)) PlotAverage_X(np.array(inference), np.array(noference)) # components = PCA.myPCA(ferences, known_cat) components = components * 1000 LOG_REG.logReg(known_cat, components) # components_tmp = components *1000 np.savetxt("eda_pcaResults.csv", np.hstack((known_cat.reshape(len(known_cat), 1), components)), delimiter=",")
def testPartition(self, traindata, traintarget, validata, valitarget, index): model = lr.LogisticRegression() w = model.fit(traindata, traintarget, lr=self.lr, eps=self.eps, num_iter=self.iterations) yp = model.predict(validata, w) self.valiError[index] = meval.evaluate_acc(yp, valitarget)
def main(): filename = 'data/default_of_credit_card_clients.xls' X, y = load_CC_data(filename) X_train, X_test, y_train, y_test = scale_data_split(X, y) model = LogisticRegression(n_epochs=1000, size_minibatch=512) model.fit(X_train, y_train) pred_model = model.predict(X_test) # clf = linear_model.LogisticRegressionCV() # clf.fit(X_train, y_train) # pred_skl = clf.predict(X_test) accuracy_model = accuracy_score(pred_model, y_test) # accuracy_skl = accuracy_score(pred_skl, y_test) print('accuracy model:', accuracy_model)
def crossValidate(data, labels, chunks, dataCont=None): ''' Perform n-fold cross validation. Given data array, labels, the chunks as folds, the function performs cross validation by using 4 out of 5 folds as training and the fifth as testing set. This is repeated 5 times with a different chunk of the fold serving as testing set. With each fold, both NB and logistic regression is fitted. The validation error with both algorithms are returned. ''' errLG = np.empty(len(chunks)) # pred error, logistic regression nIters = np.empty(len(chunks), int) # number of iterations for logistic reg errNB = np.empty(len(chunks)) # pred error, naive Bayes for ck in range(len(chunks)): # get index and dataset for current fold of cross-validation trnIdx = np.hstack([x for n, x in enumerate(chunks) if n != ck]) vldIdx = np.hstack([x for n, x in enumerate(chunks) if n == ck]) dataTrain, labelTrain = data[trnIdx], labels[trnIdx] # training dataTest, labelTest = data[vldIdx], labels[vldIdx] # validation if labels.ndim > 1: # if labels have more than one dimension labelTest = labelTest.argmax(axis=1) # collapse to 1D ## Fit and predict with naive Bayes prb = NB.NB_Train(dataTrain, labelTrain) predNB = NB.NB_Pred(dataTest, prb) errNB[ck] = errRate(predNB, labelTest) # error with naive Bayes ## Fit and predict with logistic regression if dataCont is not None: # if given non-discretized data dataTrain = dataCont[trnIdx] dataTest = dataCont[vldIdx] if labels.ndim > 1: # if more than two classes, use multinomial logistic wts, nIters[ck] = LG.fitLogisticNK(dataTrain, labelTrain, 0.5) predLG = LG.predLogisticNK(dataTest, wts) else: # binary response var, use regular logistic regression wt, nIters[ck] = LG.fitLogisticReg(dataTrain, labelTrain, 0.5) predLG = LG.predLogistic(dataTest, wt) errLG[ck] = errRate(predLG, labelTest) # error with logistic reg return errLG, errNB, nIters
def classifyVector(inX, weights): ''' LogisticRegression分类 inX:特征向量 weights:回归系数 ''' prob = LR.sigmoid(sum(inX * weights)) if prob > 0.5: return 1.0 else: return 0.0
def test_test(self): data = [ (SparseVector.SparseVector({'a': -1, 'b': -1}), -1), (SparseVector.SparseVector({'a': -1, 'b': -1}), 1), (SparseVector.SparseVector({'a':1, 'b': 1}), 1), (SparseVector.SparseVector({'a':1, 'b': 1}), 1), (SparseVector.SparseVector({'a':1, 'b': 1}), 1), (SparseVector.SparseVector({'a':1, 'b': 1}), 1) ] beta = SparseVector.SparseVector({'a': 2, 'b': 2}) scores = LogisticRegression.test(data, beta)
def trainData2(folder, gradDecent): datas, labels = LR.loadDataSet(folder, "data2.txt"); alpha = 0.1; lamb = 1; # itemNum = 500; # batch itemNum = 100; thetas = gradDecent(datas, labels, alpha, lamb, itemNum); print "thetas: ", thetas; thetas = np.mat(thetas); m = np.shape(datas)[0]; error = 0; for i in range(m): pred = LR.classfy(thetas, np.mat(datas[i])); if pred != labels[i]: error = error + 1; errorRate = float(error) * 100 / m; print "The error rate of the test is %f percents" % errorRate thetas = np.array(thetas)[0]; Plot.plot(folder, "data2.txt", thetas);
def plotBolder(theta, data, label_x, label_y, label_pos, label_neg, axes=None): plt.scatter(45, 85, s=60, c='r', marker='v', label='(45, 85)') plotData(data, label_x, label_y, label_pos, label_neg, axes) X = np.c_[np.ones((data.shape[0], 1)), data[:, :2]] x1_min, x1_max = X[:, 1].min(), X[:, 1].max(), x2_min, x2_max = X[:, 2].min(), X[:, 2].max(), xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max), np.linspace(x2_min, x2_max)) h = LR.sigmoid(np.c_[np.ones((xx1.ravel().shape[0], 1)), xx1.ravel(), xx2.ravel()].dot(theta)) h = h.reshape(xx1.shape) plt.contour(xx1, xx2, h, [0.5], linewidths=1, colors='b')
def grid_search(df): valid_size = 0.1 lamb = [('l1', 1e-9), ('l1', 1e-6), ('l1', 1e-3), ('l2', 1e-9), ('l2', 1e-6), ('l2', 1e-3), ('none', 0)] learning = [1, 0.5, 0.1, 0.01] adaptiv = ['const', 'decay'] batchsize = [10, 40, 60, 80] epochs = [1, 10, 40, 60] mi = 10**4 X, y = parse_data(df, "default payment next month", unbalanced=False) stats = pd.DataFrame(np.zeros((2 * 2 * 4**3 * 7, 8)), columns=[ "balanced", "learning rate", "adaptive learning", "epochs", "batch size", "regularization", "regularization parameter", "accuracy" ]) index = 0 for balance in [False, True]: X, y = parse_data(df, "default payment next month", unbalanced=balance) N = len(y) print("N = " + str(N) + "; 0: %.2f; 1: %.2f" % tuple(np.bincount(y) / N)) X_trian, X_eval, y_train, y_eval = train_test_split( X, y, test_size=valid_size) for l in lamb: for gamma in learning: for a in adaptiv: for bs in batchsize: for ep in epochs: clf_own = Log.LogisticRegression( max_iter=mi, mini_batch_size=bs, epochs=ep, learning_rate=gamma, adaptive_learning_rate=a, regualrization=l, logging=True) clf_own.fit(X_trian, y_train, split=True, fraction=valid_size / (1 - valid_size)) d = clf_own.evaluate(X_eval, y_eval, data_set="evaluate") stats.iloc[index] = [ balance, gamma, a, ep, bs, l[0], l[1], d["accuracy"] ] index += 1 stats.to_csv("Results/LogReg/hyper_par.csv")
def __init__(self, board): # Initialize a controller with the (empty) board information, # and what player it is, either RED or BLACK. assert playerColor == 1 or playerColor == 2 # RED or BLACK # Store player, board dimensions. # don't store actual board, just pass on each move self.playerColor = playerColor # never used self.board = board # never used self.nrows = board.getNumRows() self.ncols = board.getNumCols() # technically don't need, in board """ Initialize neural network """ self.num_hidden_units = 50 # The first hidden layer. Feed inputs directly into this. self.l_hidden = HiddenLayer(board.flattenScaleCurr(), self.nrows * self.ncols, self.num_hidden_units) # The output layer, which is a softmax layer. Note that you only need # ONE output unit, since given the input (a possible move --> # some board position), you want to estimate the probability # of winning. self.num_output_units = 1 self.l_output = LogisticRegression(self.l_hidden.output, self.l_hidden.num_output, self.num_output_units) # The error functions self.negative_log_likelihood = self.l_output.negative_log_likelihood self.errors = self.l_output.errors # Get parameters self.params = self.l_hidden.params + self.l_output.params # import os; cls=lambda: os.system("cls") # import code; code.interact(local=locals()) # Just a couple of compiled Theano functions for sigmoid and softmax. # This is for calculating forward pass. x1 = T.matrix() # some linear combination wx + b z1 = T.nnet.sigmoid(x1) self.sigmoid = theano.function([x1], z1) z2 = T.nnet.softmax(x1) # can probably take out self.softmax = theano.function([x1], z2)
def train_lr(learning_rate=9e-2, l2reg=0.0, batch_size=500, n_epochs=5000, outfile='outputs.csv'): ''' Training Otto data using Logistic Regression Create by Ghifar Tuesday, 14/04/2015 ''' print '[train_lr] learning_rate : ',learning_rate print ' l2reg : ',l2reg print ' batch_size : ',batch_size print ' n_epochs : ',n_epochs # Load training set (with contrast normalization preprocessing) # (x_train, y_train), mu, std = load_otto_data(filepath='train.csv') # (x_train, y_train), mu, std = load_otto_data(filepath='train.csv', preprocess='cn') (x_train, y_train), mu, std = load_otto_data(filepath='train.csv', preprocess='0-1') # # Load test set x_test=None y_test=None # (x_test, y_test), mu, std = load_otto_data(filepath='test.csv',preprocess='cn',mu=mu, std=std) (x_test, y_test), mu, std = load_otto_data(filepath='test.csv', preprocess='0-1') # print y_test.eval() # Contruct and compile LR model n_in = 93 n_out = 9 print x_train.eval().shape print y_train.eval().shape lr = LogisticRegression(n_in=n_in, n_out=n_out) lr.compile( train_set_x=x_train, train_set_y = y_train, #training set valid_set_x=x_train, valid_set_y = y_train, #validation set test_set_x=x_test, # test set learning_rate=learning_rate, batch_size=batch_size ) # Training with SGD lr.training_sgd(n_epochs=n_epochs) # predict print 'Predict the label...' y_predict = lr.test_model(x_test.eval()) print 'Save the prediciton: ',outfile write_otto_output(y_predict,filename=outfile) print '[train_lr] learning_rate : ',learning_rate print ' l2reg : ',l2reg print ' batch_size : ',batch_size print ' n_epochs : ',n_epochs
def test_Logistic_Regression(): iris = datasets.load_iris() X = iris.data y = iris.target X = X[y < 2, :2] y = y[y < 2] plt.scatter(X[y == 0, 0], X[y == 0, 1], c='r') plt.scatter(X[y == 1, 0], X[y == 1, 1], c='b') plt.show() X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666) log_reg = LogisticRegression() log_reg.fit(X_train, y_train) print(log_reg.score(X_test, y_test)) print(log_reg.predict_proba(X_test)) print(log_reg.predict(X_test)) print(y_test)
def testTwo(self): X = np.array( [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [16, 2, 3, 13], [5, 11, 10, 8], [9, 7, 6, 12], [4, 14, 15, 1], ] ) y = np.array([[1], [0], [1], [0], [1], [0], [1], [0]]) theta = np.array([[0], [1], [0], [1]]) expectedJ = 4.8135 expectedTheta = np.array([[1.3154], [3.3154], [3.3154], [1.3154]]) [actualJ, actualTheta] = LR.costFunction(X, y, theta) np.testing.assert_almost_equal(actualJ, expectedJ, decimal=4) np.testing.assert_almost_equal(actualTheta, expectedTheta, decimal=4)
def __init__(self, rng, input, n_in, n_out, n_layers, units): # build and connect each hidden layer assert n_layers == len(units) self.hidden_layers = [] self.hidden_layers.append( HiddenLayer(rng=rng, input=input, n_in=n_in, n_out=units[0], activation=T.tanh)) i = 1 while i < n_layers: prev_layer = self.hidden_layers[-1] layer = HiddenLayer(rng=rng, input=prev_layer.output, n_in=units[i - 1], n_out=units[i], activation=T.tanh) self.hidden_layers.append(layer) i += 1 # logistic regression layer for classification self.softmax_layer = LogisticRegression( input=self.hidden_layers[-1].output, n_in=units[-1], n_out=n_out) self.L1 = sum( [abs(self.hidden_layers[i].W).sum() for i in xrange(n_layers)]) + abs(self.softmax_layer.W).sum() self.L2_sqr = sum([(self.hidden_layers[i].W ** 2).sum() for i in xrange(n_layers)]) + \ (self.softmax_layer.W ** 2).sum() self.nll = self.softmax_layer.nll self.errors = self.softmax_layer.errors self.make_shared_dataset = self.softmax_layer.make_shared_dataset self.params = self.hidden_layers[0].params for i in xrange(1, n_layers): self.params += self.hidden_layers[i].params self.params += self.softmax_layer.params self.input = input self.ypred = self.softmax_layer.ypred
def __testCore(trainingData, testData, optimizer): lr = LogisticRegression.LogisticRegression(optimizer) costValue = lr.train(trainingData[:, :-1], trainingData[:, -1]) lr.sigLevel = None print("theta: {0}, value of cost: {1}".format(lr.theta, costValue)) actualValue = testData[:, -1] predictValue = (lr.predictValue(testData[:, :-1]) > 0.5) - 0 tp = predictValue[(actualValue == 1).A.flatten(), :].sum() fp = predictValue[(actualValue == 0).A.flatten(), :].sum() tn = -(predictValue - 1)[(actualValue == 0).A.flatten(), :].sum() fn = -(predictValue - 1)[(actualValue == 1).A.flatten(), :].sum() accuracy = (tp + tn) / (tp + fp + tn + fn) precision = tp / (tp + fp) recall = tp / (tp + fn) f1 = 2 * tp / (2 * tp + fp + fn) print("accuracy: {0}, precision: {1}, recall: {2}, f1: {3}".format( accuracy, precision, recall, f1))
def k_fold(data, k): np.random.shuffle(data) data_subsets = np.array_split(data, k, axis=0) avg_acc = 0 start = time.time() for i in range(k): training_data = np.concatenate(data_subsets[:i] + data_subsets[i + 1:], axis=0) validation_data = data_subsets[i] logistic_regression_k_fold = LR.LogisticRegression( training_data[:, 0:-1], training_data[:, -1]) logistic_regression_k_fold.fit(start_learning_rate, end_learning_rate, gradient_descent_iterations) y_perdict = logistic_regression_k_fold.predict(validation_data[:, 0:-1]) avg_acc = avg_acc + evaluate_acc( validation_data[:, -1].reshape(validation_data.shape[0], 1), y_perdict) print(avg_acc / k) print("Logestic average accurrcy: {0:.2%}".format(avg_acc / k)) print("Time cost: ", time.time() - start) start = time.time() avg_acc = 0 for i in range(k): training_data = np.concatenate(data_subsets[:i] + data_subsets[i + 1:], axis=0) validation_data = data_subsets[i] linear_discriminant_analysis_k_fold = LDA.LinearDiscriminantAnalysis( training_data[:, 0:-1], training_data[:, -1]) linear_discriminant_analysis_k_fold.fit() y_perdict = linear_discriminant_analysis_k_fold.predict( validation_data[:, 0:-1]) avg_acc = avg_acc + evaluate_acc( validation_data[:, -1].reshape(validation_data.shape[0], 1), y_perdict) print("LDA average accurrcy: {0:.2%}".format(avg_acc / k)) print("Time cost: ", time.time() - start) return
def __init__(self, norm_type="Normalization", iterations=5, base_classifier="SVM"): super().__init__() self.iterations = iterations self.norm_type = norm_type # self.prediction = None # self.probability = None self.classifier_set = None if base_classifier == "SVM": self.base_classifier = SVM.SVMClassifier() elif base_classifier == "KNN": self.base_classifier = KNN.KNNClassifier() elif base_classifier == "DecisionTree": self.base_classifier = DecisionTree.DecisionTreeClassifier() elif base_classifier == "Logistic": self.base_classifier = LogisticRegression.LogisticRegressionClassifier( ) elif base_classifier == "Perceptron": self.base_classifier = Perceptron.PerceptronClassifier() super().__init__()
def buildAdv(): rng = np.random.RandomState() print "Loading Data..." [training_data_shared, training_categories_shared, validation_data_shared, validation_categories_shared, test_data_shared, test_categories_shared] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES, TEST_DIR, NUM_TEST_EXAMPLES) current_generated_data_shared = theano.shared(np.zeros_like(training_data_shared.get_value())) current_generated_categories_shared = theano.shared(np.int32(np.zeros_like(training_categories_shared.get_value()))) print "Building the model..." # we want a generative mlp that goes from a category and a random seed, through three hidden layers, to a 28x28 image xg_category = T.cast(T.ivector('xg_category'), 'int32') # input category xg_seed = T.matrix('xg_seed') xg_category_onehot = T.extra_ops.to_one_hot(xg_category, 10) # first mlp hidden layer """""" glayer0 = HiddenLayer(rng, input=T.concatenate((xg_category_onehot, xg_seed), axis=1), n_in=20, n_out=100, activation=T.tanh) # second mlp hidden layer glayer1 = HiddenLayer(rng, input=glayer0.output, n_in=100, n_out=500, activation=T.tanh) glayer2 = HiddenLayer(rng, input=glayer1.output, n_in=500, n_out=1000, activation=T.tanh) # last mlp hidden layer glayer3 = HiddenLayer(rng, input=glayer2.output, n_in=1000, n_out=28*28, activation=T.tanh) # the output of the generative network goutput = glayer3.output.reshape((batch_size, 1, 28,28)) # we want a discriminative mlp that goes from an image, through a convolutional layer, through a hidden layer, through a logistic regression to decide whether xd = glayer3.output # input images yd = xg_category # label for the images yd_onehot = T.extra_ops.to_one_hot(yd, 10) zd = T.ivector('z') # real or not dlayer0Input = xd.reshape((batch_size, 1, 28, 28)) # reassemble the image # convolutional layer dlayer0 = ConvPoolingLayer(rng, input=dlayer0Input, image_shape=(batch_size, 1, 28, 28), filter_shape=(20, 1, 5, 5), poolsize=(2,2)) # first mlp hidden layer dlayer1 = HiddenLayer(rng, input=dlayer0.output.flatten(2), n_in=20*9*4*4, n_out=500, activation=T.tanh) # second mlp hidden layer dlayer2 = HiddenLayer(rng, input=T.concatenate((dlayer1.output, yd_onehot), axis=1), n_in=510, n_out=100, activation=T.tanh) # logistic regression layer - note the dimension magic needed to append the category to each input vector dlayer3 = LogisticRegression(input=dlayer2.output, n_in=100, n_out=2) # 0 is generated, 1 is real dcost = dlayer3.negative_log_likelihood(zd) # This approach uses the negative log liklihood that the incorrect answer is selected (p_y_given_x is the softmax of the output of the layer) gcost = -T.mean(T.log(dlayer3.p_y_given_x)[T.arange(zd.shape[0]), 1-zd]) # using the given for zd dparams = dlayer3.params + dlayer2.params + dlayer1.params + dlayer0.params gparams = glayer3.params + glayer2.params + glayer1.params + glayer0.params dgrads = T.grad(dcost, dparams) ggrads = T.grad(gcost, gparams) dopt = rmsprop(dparams) gopt = rmsprop(gparams) dupdates = dopt.updates(dparams, dgrads, learning_rate / np.cast['float32'](batch_size), momentum) gupdates = gopt.updates(gparams, ggrads, learning_rate / np.cast['float32'](batch_size), momentum) # functions to train the discriminative model # important: call gfire first and set current_generated_data_shared and current_generated_categories_shared to the returned values before training zd_temp = np.zeros(batch_size) zd_temp[:batch_size//2] = 1 zd_given = theano.shared(np.int32(zd_temp), borrow=True) index = T.lscalar() # training occurs on both real and generated training data dtrain_model = theano.function([index], dcost, updates=dupdates, givens={xd:T.concatenate((training_data_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])), yd:T.concatenate((training_categories_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])), zd:zd_given}) # validation is just on the validation data - it sees how many of them it thinks are real dvalidate_model = theano.function([index], dlayer3.errors(zd), givens={xd:validation_data_shared[index * batch_size: (index+1) * batch_size], yd:validation_categories_shared[index * batch_size: (index+1) * batch_size], zd:theano.shared(np.int32(np.ones(batch_size)))}) # testing occurs on real and generated dtest_model = theano.function([index], dlayer3.errors(zd), givens={xd:T.concatenate((test_data_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])), yd:T.concatenate((test_categories_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])), zd:zd_given}) sample = T.matrix('samples') category = T.ivector('category') dfire = theano.function([sample, category], dlayer3.p_y_given_x, givens={xd:sample, yd:category}) # category values and random numbers - input to generative network ginput_categories_shared = theano.shared(np.int32(np.array(range(10)*(batch_size//10)))) ginput_rng_shared = theano.shared(rng.uniform(low=-1,high=1, size=[batch_size,10])) # function to train the generative model gtrain_model = theano.function([], gcost, updates=gupdates, givens={xg_category:ginput_categories_shared, xg_seed:ginput_rng_shared, zd:theano.shared(np.int32(np.zeros(batch_size)))}) # function to test/validate the generative model gtest_model = theano.function([], 1-dlayer3.errors(zd), givens={xg_category:ginput_categories_shared, xg_seed:ginput_rng_shared,zd:theano.shared(np.int32(np.zeros(batch_size)))}) # function to generate new training examples for the discriminative model gfire = theano.function([], [goutput, xg_category], givens={xg_category:ginput_categories_shared, xg_seed:ginput_rng_shared}) dbest_loss = 1 def generate_new_data(): ginput_rng_shared.set_value(rng.uniform(low=0,high=1, size=[batch_size,10])) generated_data = np.zeros([NUM_TRAINING_EXAMPLES,1,28,28]) generated_categories = np.zeros_like(current_generated_categories_shared.get_value()) for i in range(NUM_TRAINING_EXAMPLES//batch_size): generated_data[batch_size * i: batch_size * (i+1)], generated_categories[batch_size * i: batch_size * (i+1)] = gfire() current_generated_data_shared.set_value(generated_data.reshape([NUM_TRAINING_EXAMPLES,28*28])) current_generated_categories_shared.set_value(generated_categories) # training loop print("Training...") for epoch in range(1, num_epochs): print "training epoch: ", epoch if epoch % 10 == 0: print "evaluating discriminator..." # fire the generator before validating generate_new_data() # calculate validation loss dvalidation_loss = [dvalidate_model(i) for i in range(NUM_VALIDATION_EXAMPLES//batch_size)] mean_dvalidation_loss = np.mean(dvalidation_loss) print"discrimination error (% of real labeled fake): ", mean_dvalidation_loss * 100, "%" print "evaluating generator..." gvalidation_loss = gtest_model() print"generation quality (% of fake labeled real): ", (1-gvalidation_loss) * 100, "%" if epoch % 10 == 0: for i in range(10): #print "percent certainty that generated image is real: ", dfire(current_generated_data_shared[:batch_size].reshape([batch_size,28*28]),current_generated_categories_shared[:batch_size])[i] plt.imshow(np.reshape(current_generated_data_shared.get_value()[i], [28,28]), cmap=plt.get_cmap('gray'), interpolation='nearest') #plt.show() plt.savefig("output2/epoch" + str(epoch) + "image" + str(current_generated_categories_shared.get_value()[i]) + ".png") # fire the generator before training ginput_rng_shared = theano.shared(rng.uniform(low=0,high=1, size=[batch_size,10])) generate_new_data() dc = 1 gc = 1 for batch_index in range(NUM_TRAINING_EXAMPLES//batch_size * 2): if gc*.1 < dc : dc = dtrain_model(batch_index) if batch_index % (NUM_TRAINING_EXAMPLES//batch_size) == 0: print "dcost: ", dc test = dfire(np.append(training_data_shared.get_value()[0: batch_size//2], current_generated_data_shared.get_value()[0: batch_size//2], axis=0), np.append(training_categories_shared.get_value()[0: batch_size//2], current_generated_categories_shared.get_value()[0:batch_size//2], axis=0)) print "percent certainty that real image is [fake,real]: ", test[0] print "correct answer: ", zd_given.get_value()[0] print "percent certainty that generated image is [fake,real]: ", test[batch_size//2] print "correct answer: ", zd_given.get_value()[batch_size//2] if dc*.1 < gc : ginput_rng_shared = theano.shared(rng.uniform(low=0,high=1, size=[batch_size,10])) gc = gtrain_model() if batch_index % (NUM_TRAINING_EXAMPLES//batch_size) == 0: print "gcost: ", gc plt.show()
def loadData(): train_x = [] train_y = [] fileIn = open('testSet_LR.txt') for line in fileIn.readlines(): lineArr = line.strip().split() train_x.append([1.0, float(lineArr[0]), float(lineArr[1])]) train_y.append(float(lineArr[2])) return mat(train_x), mat(train_y).transpose() ## step 1: load data print "step 1: load data..." train_x, train_y = loadData() test_x = train_x; test_y = train_y ## step 2: training... print "step 2: training..." opts = {'alpha': 0.01, 'maxIter': 1000, 'optimizeType': 'gradDescent'} optimalWeights = lr.trainLogRegres(train_x, train_y, opts) ## step 3: testing print "step 3: testing..." accuracy = lr.testLogRegres(optimalWeights, test_x, test_y) ## step 4: show the result print "step 4: show the result..." print 'The classify accuracy is: %.3f%%' % (accuracy * 100) lr.showLogRegres(optimalWeights, train_x, train_y)
import csv import numpy as np csv_file_object = csv.reader(open('train.csv', 'rb')) #Load in the csv file header = csv_file_object.next() #Skip the fist line as it is a header data=[] #Creat a variable called 'data' for row in csv_file_object: #Skip through each row in the csv file data.append(row) #adding each row to the data variable data = np.array(data) #Then convert from a list to an array #print header #survived0,pclass1-0,name2,sex3-1,age4-2,sibsp5-3,parch6-4,ticket7,fare8-5,cabin9,embarked10 outputs = np.matrix(data[0:, 0]).T.astype(int) inputs = data[0:, (1, 3, 4, 5, 6, 8)] inputs[:, 1] = lr.eapply(lambda x: int(x), data[0::,3] == "male") def liq(x): if x == '': return 0 else: return x inputs = np.matrix(lr.eapply(liq, inputs)) inputs = np.matrix(inputs).T.astype(float) def clasterize(vector, classCount): a = list(np.array(vector).reshape(-1)) maxv, minv = max(a), min(a) h = float(maxv - minv) / classCount return [ int((x - minv) / h) for x in a] classCount = 4 clasterized = (2, 3, 5)
import csv import numpy as np csv_file_object = csv.reader(open('train.csv', 'rb')) #Load in the csv file header = csv_file_object.next() #Skip the fist line as it is a header data=[] #Creat a variable called 'data' for row in csv_file_object: #Skip through each row in the csv file data.append(row) #adding each row to the data variable data = np.array(data) #Then convert from a list to an array #print header #survived0,pclass1-0,name2,sex3-1,age4-2,sibsp5-3,parch6-4,ticket7,fare8-5,cabin9,embarked10 outputs = np.matrix(data[0:, 0]).T.astype(int) inputs = data[0:, (1, 3, 4, 5, 6, 8)] inputs[:, 1] = lr.eapply(lambda x: int(x), data[0::,3] == "male") def liq(x): if x == '': return 0 else: return x inputs = np.matrix(lr.eapply(liq, inputs)) inputs = np.matrix(inputs).T.astype(float) w, e = lr.batchGradientDescent(inputs, outputs, np.matrix([[0] * 7]).T, 0.5, -0.01, 100) print e[0:4], '...' print e[-5:-1] """s = 0.0 for m in range(outputs.shape[0]):
def classify(): [training_data_shared, training_categories_shared, validation_data_shared, validation_categories_shared, test_data_shared, test_categories_shared] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES, TEST_DIR, NUM_TEST_EXAMPLES) # create a nn with two 28**2 5x5 convolutional layer (each node looks at 25 pixels in a square) + convolutional pooling layer, then a regular nn layer print"building the model..." rng = np.random.RandomState() nkerns=[20,50] batch_size = 100 n_train_batches = NUM_TRAINING_EXAMPLES/batch_size x = T.matrix('x') y = T.ivector('y') index = T.lscalar() layer0_input = x.reshape((batch_size, 1, 28, 28)) layer0 = ConvPoolingLayer(rng, input=layer0_input, image_shape=(batch_size, 1, 28, 28), filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2,2)) # outputs a convoluted image layer1 = ConvPoolingLayer(rng, input=layer0.output, image_shape=(batch_size, nkerns[0], 12, 12), filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2,2)) # outputs a doubly convoluted image # flatten before putting it through the rest of the NN layer2_input = layer1.output.flatten(2) layer2 = HiddenLayer(rng, input=layer2_input, n_in=nkerns[1]*4*4, n_out=500, activation=T.tanh) # tanh activation function (sigmoidal) layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10) cost = layer3.negative_log_likelihood(y) test_model = theano.function([index], layer3.errors(y), givens={x:test_data_shared[index * batch_size: (index+1) * batch_size], y:test_categories_shared[index * batch_size: (index+1) * batch_size]}) validate_model = theano.function([index], layer3.errors(y), givens={x:validation_data_shared[index * batch_size: (index+1) * batch_size], y:validation_categories_shared[index * batch_size: (index+1) * batch_size]}) # train on the images print"training..." params = layer3.params + layer2.params + layer1.params + layer0.params grads = T.grad(cost, params) # the magic step updates = [(param_i, param_i - learning_rate * grad_i) for param_i, grad_i in zip(params, grads)] train_model = theano.function([index], cost, updates=updates, givens={x:training_data_shared[index * batch_size: (index+1) * batch_size], y:training_categories_shared[index * batch_size: (index+1) * batch_size]}) epoch = 0 best_loss = 1 while (epoch < N_EPOCHS): epoch +=1 if epoch %5 == 0: print"training epoch: ", epoch if epoch %5 == 0: print"validating..." validation_loss = [validate_model(i) for i in range(NUM_VALIDATION_EXAMPLES/batch_size)] this_validation_loss = np.mean(validation_loss) if this_validation_loss < best_loss: best_loss = this_validation_loss else: print "loss is worse now, breaking..." break print"loss: ", this_validation_loss * 100, "%" for minibatch_index in range(n_train_batches): train_model(minibatch_index) print"done training! Testing..." test_losses = [test_model(i) for i in range(NUM_TEST_EXAMPLES/batch_size)] test_score = np.mean(test_losses) print"Test score loss: ", test_score * 100, "%"
return self.responses_test def get_training_predictors(self): """ Gets the training predictors. :return: The training predictors """ return self.predictors_training def get_training_responses(self): """ Gets the training responses. :return: The training responses """ return self.responses_training def get_training_sample_count(self): """ Gets the number of training samples. :return: The number of training samples """ return self.training_sample_count # Create a spam data set object, print a start message, and give a # demonstration of hand-coded logistic regression using the spam # data set! DATA_SET = SpamData() print("Demonstration of hand-coded logistic gradient descent versus " "scikit-learn's implementation using the spam data set...") LogisticRegression.give_demonstration(DATA_SET)
def testThree(self): z = np.array([[8, 1, 6], [3, 5, 7], [4, 9, 2]]) expected = np.array([[0.9997, 0.7311, 0.9975], [0.9526, 0.9933, 0.9991], [0.9820, 0.9999, 0.8808]]) np.testing.assert_almost_equal(LR.sigmoid(z), expected, decimal=4)
def plot_learning_curve(features_train, labels_train, features_test, labels_test, outputClasses=None, K="linear", C=1, G=0.01, method="error", classifier="SVM", a=0, b=1, showFigure=True, saveFigure=False, savePath=None, nb_epochs=10): # run for every 10% of training set and compute training error and testing error step = len(features_train) / 10 train = [] test = [] maj_clas = [] for i in range(0, 10): print 'iteration : ', i # train for (i+1)*10 percent of training set f = features_train[0:((i + 1) * (step))] l = labels_train[0:((i + 1) * (step))] assert f.shape[0] == l.shape[0], 'Wrong number of input data! ' if classifier == "SVM": # train classifier for the specific subset of training set model = SVM.train(f, l, k=K, c=C, g=G) # get training error predictionTrain = SVM.predict(f, model) # get testing error predictionTest = SVM.predict(features_test, model) elif classifier == "LR": # train classifier for the specific subset of training set model = LogisticRegression.train(f, l, c=C) # get training error predictionTrain = LogisticRegression.predict(f, model) # get testing error predictionTest = LogisticRegression.predict(features_test, model) elif classifier == "CNN": model = Keras_test.CNN().train(features=f, labels=l,outputClasses=outputClasses, learning_curves_OR_Cross_Val=True,nb_epoch=nb_epochs) # get training error predictionTrain = Keras_test.CNN().predictClasses(features=f, model=model) # get testing error predictionTest = Keras_test.CNN().predictClasses(features=features_test, model=model) # TODO : CNN MINIBATCHES LEARNING CURVES , Implementation : read 10% of data and train cnn.train with all the data # elif classifier == "CNN_minibatches": elif classifier == "MLP": model = Keras_test.MLP().train(features=f, labels=l, outputClasses=outputClasses, learning_curves_OR_Cross_Val=True, nb_epoch=nb_epochs) # get training error predictionTrain = Keras_test.MLP().predict(features=f, model=model, ShowAccuracy=False) # get testing error predictionTest = Keras_test.MLP().predict(features=features_test, model=model, ShowAccuracy=False) elif classifier == "SimpleRNN": model = Keras_test.RNN().trainRNN(features=f, labels=l, outputClasses=outputClasses, learning_curves=True) # get training error predictionTrain = Keras_test.RNN().predict(features=f, model=model, ShowAccuracy=False) # get testing error predictionTest = Keras_test.RNN().predict(features=features_test, model=model, ShowAccuracy=False) elif classifier == "RNN_LSTM": model = Keras_test.RNN().trainRnnLSTM(features=f, labels=l, outputClasses=outputClasses, learning_curves=True) # get training error predictionTrain = Keras_test.RNN().predict(features=f, model=model, ShowAccuracy=False) # get testing error predictionTest = Keras_test.RNN().predict(features=features_test, model=model, ShowAccuracy=False) # get error for majority classifier predictionMajority = MajorityClassifier.predictMaj(labels_test) if method == "error": train.append(measures.error(l, predictionTrain)) test.append(measures.error(labels_test, predictionTest)) maj_clas.append(measures.error(labels_test, predictionMajority)) elif method == "avgF1": train.append(measures.avgF1(l, predictionTrain, a, b)) test.append(measures.avgF1(labels_test, predictionTest, a, b)) maj_clas.append(measures.avgF1(labels_test, predictionMajority, a, b)) print test[9] x = np.arange(len(train)) * 10 plt.plot(x, train, color="blue", linewidth="2.0", label=classifier) plt.plot(x, test, color="blue", linestyle="dashed", linewidth="2.0") plt.plot(x, maj_clas, color="red", linewidth="2.0") plt.ylim(0, 1) plt.ylabel(method) plt.xlabel("% of messages") if method == "error": plt.legend(loc="upper left") elif method == "avgF1": plt.legend(loc="lower left") if saveFigure: assert savePath != None, "Give image path to save image" # with figure i can save it anywhere i want # fig1 = plt.gcf() plt.savefig(savePath) # clear current canvas . if we have show and save together we will have a problem... plt.clf() if showFigure: plt.show()
def buildAdversarialGA(): rng = np.random.RandomState() print "Loading Data..." [training_data_shared, training_categories_shared, validation_data_shared, validation_categories_shared, test_data_shared, test_categories_shared] = loadImages(TRAIN_DIR, NUM_TRAINING_EXAMPLES, NUM_VALIDATION_EXAMPLES, TEST_DIR, NUM_TEST_EXAMPLES) current_generated_data_shared = theano.shared(np.zeros_like(training_data_shared.get_value())) current_generated_categories_shared = theano.shared(np.int32(np.zeros_like(training_categories_shared.get_value()))) print "Building the model..." xd = T.matrix('xd') # input images yd = T.ivector('yd') # label for the images yd_onehot = T.extra_ops.to_one_hot(yd, 10) zd = T.ivector('z') # real or not dlayer0Input = xd.reshape((batch_size, 1, 28, 28)) # reassemble the image # convolutional layer dlayer0 = ConvPoolingLayer(rng, input=dlayer0Input, image_shape=(batch_size, 1, 28, 28), filter_shape=(20, 1, 5, 5), poolsize=(2,2)) # first mlp hidden layer dlayer1 = HiddenLayer(rng, input=dlayer0.output.flatten(2), n_in=20*9*4*4, n_out=500, activation=T.tanh) # second mlp hidden layer dlayer2 = HiddenLayer(rng, input=T.concatenate((dlayer1.output, yd_onehot), axis=1), n_in=510, n_out=100, activation=T.tanh) # logistic regression layer - note the dimension magic needed to append the category to each input vector dlayer3 = LogisticRegression(input=dlayer2.output, n_in=100, n_out=2) # 0 is generated, 1 is real dcost = dlayer3.negative_log_likelihood(zd) dparams = dlayer3.params + dlayer2.params + dlayer1.params + dlayer0.params dgrads = T.grad(dcost, dparams) dopt = rmsprop(dparams) dupdates = dopt.updates(dparams, dgrads, learning_rate / np.cast['float32'](batch_size), momentum) zd_temp = np.zeros(batch_size) zd_temp[:batch_size//2] = 1 zd_given = theano.shared(np.int32(zd_temp), borrow=True) index = T.lscalar() # training occurs on both real and generated training data dtrain_model = theano.function([index], dcost, updates=dupdates, givens={xd:T.concatenate((training_data_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])), yd:T.concatenate((training_categories_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])), zd:zd_given}) # validation is just on the validation data - it sees how many of them it thinks are real dvalidate_model = theano.function([index], dlayer3.errors(zd), givens={xd:validation_data_shared[index * batch_size: (index+1) * batch_size], yd:validation_categories_shared[index * batch_size: (index+1) * batch_size], zd:theano.shared(np.int32(np.ones(batch_size)))}) # testing occurs on real and generated dtest_model = theano.function([index], dlayer3.errors(zd), givens={xd:T.concatenate((test_data_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_data_shared[index * batch_size//2: (index+1) * batch_size//2])), yd:T.concatenate((test_categories_shared[index * batch_size//2: (index+1) * batch_size//2], current_generated_categories_shared[index * batch_size//2: (index+1)* batch_size//2])), zd:zd_given}) sample = T.matrix('samples') category = T.ivector('category') real = T.ivector('real?') # fire the discriminator on input samples, what category they are trying to be and whether they're real or generated (1 == real) dfire = theano.function([sample, category], dlayer3.p_y_given_x, givens={xd:sample, yd:category}) # fitness function needs to accomidate a list of genes - currently it only accepts batch_size at once, so it loops def fitness_function(a): retval = np.array([]) for i in range(len(a)//batch_size): retval = np.append(retval, dfire(np.packbits(a, axis=1)[i*batch_size: (i+1) * batch_size, :], [digit_to_generate]*batch_size)[:,1]) return retval current_generated_categories_shared.set_value([digit_to_generate]*NUM_TRAINING_EXAMPLES) gmodel = pvga(fitness_function, 28*28*8, rng, num_samples_per_gen=2000) mutation_rate = .05 print "Training..." for epoch in range(1, num_epochs): print "Epoch: ", epoch if epoch %10 == 0: #print "Validating..." #error = np.array([]) #for batch_index in range(NUM_VALIDATION_EXAMPLES//batch_size *2): #error = np.append(error, dvalidate_model(batch_index)) #print "D error: ", np.mean(error) * 100, "%" plt.imshow(np.reshape(current_generated_data_shared.get_value()[0],[28,28]), cmap=plt.get_cmap('gray'), interpolation='nearest') plt.draw() data = np.packbits(gmodel.sample_pop(NUM_TRAINING_EXAMPLES),axis=1) current_generated_data_shared.set_value(data.reshape((NUM_TRAINING_EXAMPLES, 28*28))) print "Training D:" loss = np.array([]) for batch_index in range(NUM_TRAINING_EXAMPLES//batch_size * 2): loss = np.append(loss, dtrain_model(batch_index)) print "Average loss: ", np.mean(loss) print "Training G:" gmodel.create_next_generation(mutation_rate=mutation_rate) prev_max = gmodel.max_fitness for i in range(epoch * 50): #mutation_rate/=2 print "Max fitness: ", gmodel.max_fitness gmodel.create_next_generation(mutation_rate=mutation_rate) prev_max = gmodel.max_fitness plt.show()
def testOne(self): np.testing.assert_almost_equal(LR.sigmoid(1200000), 1) np.testing.assert_almost_equal(LR.sigmoid(-250), 0) np.testing.assert_almost_equal(LR.sigmoid(0), 0.5)
yield (inputs[p])[idx:idx + batchsize], (targets[p])[idx:idx + batchsize] print("Loading the data") X_train, y_train, X_test, y_test = mnist.load_dataset(args['nbtrain'], args['nbtest'], False) ############################################################################ # Model building print("Model architecture : ") filename_prefix = "" if ('logreg' in args): print(" Logistic regression, Input -> Softmax") model = LogisticRegression.Model(28, 28, 1, 10) filename_prefix += "logreg_" elif 'mlp' in args: print(" Multilayer perceptron with : ") filename_prefix += "mlp_" sys.exit() elif 'cnn' in args: print(" Convolutional neural network with :") filename_prefix += "cnn_" sys.exit() elif 'vgg' in args: print(" Oxford Visual Geometry Group with :") filename_prefix += "vgg_" sys.exit() else: sys.exit()
import numpy as np from random import seed, gauss import LogisticRegression as lr i = np.matrix([ [1,1], [1,2], [1,3], [2,1], [2,2], [2,3], [3,1], [3,2], [3,3] ]).T o = np.matrix([[0, 0, 1, 0, 1, 1, 1, 1, 1]]).T w = lr.batchGradientDescent( lr.LogisticDerivative, i, o, np.matrix([[0], [0], [0]]), 0.5, -0.5, 70) print w for i in xrange(3): for v in xrange(3): print i, v, lr.calcLogReg(w, np.matrix([i+1,v+1]).T, 0.5) print lr.calcLogReg(w, np.matrix([3,0.85352157]).T, 0.5)
test_output_labels[:, :, i] = datum.label() numberOfData = train_input_data.shape[2] normalizeInputSum = np.sum(train_input_data, axis=2) normalizeInputSum = np.tile(normalizeInputSum, [numberOfData, 1, 1]).transpose(1, 2, 0) train_input_data = np.divide(np.power(train_input_data, 2), normalizeInputSum) numberOfData = test_input_data.shape[2] normalizeInputSum = np.sum(test_input_data, axis=2) normalizeInputSum = np.tile(normalizeInputSum, [numberOfData, 1, 1]).transpose(1, 2, 0) test_input_data = np.divide(np.power(test_input_data, 2), normalizeInputSum) # V, W = NeuralNetwork.trainNetwork(train_input_data[:, :, 0:100], train_output_labels[0:100], 100, 8, 0.0000001) lrV = LogisticRegression.trainNetwork(train_input_data, train_output_labels, 100, 0.01); lrV2 = LogisticRegression2.trainNetwork(train_input_data[0:100], train_output_labels[0:100], 0.001) # np.save('data/V-3', V) # np.save('data/W-3', W) np.save('data/lrV', lrV) np.save('data/lrV2', lrV2) # V = np.load('data/V-2.npy') # W = np.load('data/W-2.npy') # error, labels = NeuralNetwork.testNetwork(V, W, test_input_data[:, :, 4000:6000], test_output_labels[:, :, 4000:6000], 8) # test_labels = np.argmax(test_output_labels[:, :, 4000:6000].reshape([2, 2000]), axis=0) # print np.sum(test_labels == labels)
import numpy as np import matplotlib.pyplot as plt from sklearn import datasets iris = datasets.load_iris() X = iris.data y = iris.target print(X,y) X = X[y<2,:2] y = y[y<2] X.shape (100, 2) y.shape (100,) plt.scatter(X[y==0,0], X[y==0,1], color="red") plt.scatter(X[y==1,0], X[y==1,1], color="blue") plt.show() from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_test = train_test_split(X, y) import LogisticRegression log_reg = LogisticRegression() log_reg.fit(X_train, y_train) LogisticRegression() log_reg.score(X_test, y_test) log_reg.predict_proba(X_test) log_reg.predict(X_test)
class DBN(object): """Deep Belief Network A deep belief network is obtained by stacking several RBMs on top of each other. The hidden layer of the RBM at layer `i` becomes the input of the RBM at layer `i+1`. The first layer RBM gets as input the input of the network, and the hidden layer of the last RBM represents the output. When used for classification, the DBN is treated as a MLP, by adding a logistic regression layer on top. """ def __init__(self, numpy_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10): """This class is made to support a variable number of layers. :type numpy_rng: numpy.random.RandomState :param numpy_rng: numpy random number generator used to draw initial weights :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` :type n_ins: int :param n_ins: dimension of the input to the DBN :type hidden_layers_sizes: list of ints :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int :param n_outs: dimension of the output of the network """ self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 if not theano_rng: theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30)) # allocate symbolic variables for the data self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector # of [int] labels # end-snippet-1 # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first # construct the DBN as a deep multilayer perceptron, and when # constructing each sigmoidal layer we also construct an RBM # that shares weights with that layer. During pretraining we # will train these RBMs (which will lead to chainging the # weights of the MLP as well) During finetuning we will finish # training the DBN by doing stochastic gradient descent on the # MLP. for i in range(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden # units of the layer below or the input size if we are on # the first layer if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # the input to this layer is either the activation of the # hidden layer below or the input of the DBN if you are on # the first layer if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=numpy_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... but we are # going to only declare that the parameters of the # sigmoid_layers are parameters of the DBN. The visible # biases in the RBM are parameters of those RBMs, but not # of the DBN. self.params.extend(sigmoid_layer.params) # Construct an RBM that shared weights with this layer rbm_layer = RBM(numpy_rng=numpy_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We now need to add a logistic layer on top of the MLP self.logLayer = LogisticRegression( input=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined as the # negative log likelihood of the logistic regression (output) layer self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logLayer.errors(self.y) def pretraining_functions(self, train_set_x, batch_size, k): '''Generates a list of functions, for performing one step of gradient descent at a given layer. The function will require as input the minibatch index, and to train an RBM you just need to iterate, calling the corresponding function on all minibatch indexes. :type train_set_x: theano.tensor.TensorType :param train_set_x: Shared var. that contains all datapoints used for training the RBM :type batch_size: int :param batch_size: size of a [mini]batch :param k: number of Gibbs steps to do in CD-k / PCD-k ''' # index to a [mini]batch index = T.lscalar('index') # index to a minibatch learning_rate = T.scalar('lr') # learning rate to use # number of batches n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size # begining of a batch, given `index` batch_begin = index * batch_size # ending of a batch given `index` batch_end = batch_begin + batch_size pretrain_fns = [] for rbm in self.rbm_layers: # get the cost and the updates list # using CD-k here (persisent=None) for training each RBM. # TODO: change cost function to reconstruction error cost, updates = rbm.get_cost_updates(learning_rate, persistent=None, k=k) # compile the theano function fn = theano.function( inputs=[index, theano.In(learning_rate, value=0.1)], outputs=cost, updates=updates, givens={ self.x: train_set_x[batch_begin:batch_end] } ) # append `fn` to the list of functions pretrain_fns.append(fn) return pretrain_fns def build_finetune_functions(self, datasets, batch_size, learning_rate): '''Generates a function `train` that implements one step of finetuning, a function `validate` that computes the error on a batch from the validation set, and a function `test` that computes the error on a batch from the testing set :type datasets: list of pairs of theano.tensor.TensorType :param datasets: It is a list that contain all the datasets; the has to contain three pairs, `train`, `valid`, `test` in this order, where each pair is formed of two Theano variables, one for the datapoints, the other for the labels :type batch_size: int :param batch_size: size of a minibatch :type learning_rate: float :param learning_rate: learning rate used during finetune stage ''' (train_set_x, train_set_y) = datasets[0] (valid_set_x, valid_set_y) = datasets[1] (test_set_x, test_set_y) = datasets[2] # compute number of minibatches for training, validation and testing n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] n_valid_batches /= batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] n_test_batches /= batch_size index = T.lscalar('index') # index to a [mini]batch # compute the gradients with respect to the model parameters gparams = T.grad(self.finetune_cost, self.params) # compute list of fine-tuning updates updates = [] for param, gparam in zip(self.params, gparams): updates.append((param, param - gparam * learning_rate)) train_fn = theano.function( inputs=[index], outputs=self.finetune_cost, updates=updates, givens={ self.x: train_set_x[ index * batch_size: (index + 1) * batch_size ], self.y: train_set_y[ index * batch_size: (index + 1) * batch_size ] } ) test_score_i = theano.function( [index], self.errors, givens={ self.x: test_set_x[ index * batch_size: (index + 1) * batch_size ], self.y: test_set_y[ index * batch_size: (index + 1) * batch_size ] } ) valid_score_i = theano.function( [index], self.errors, givens={ self.x: valid_set_x[ index * batch_size: (index + 1) * batch_size ], self.y: valid_set_y[ index * batch_size: (index + 1) * batch_size ] } ) # Create a function that scans the entire validation set def valid_score(): return [valid_score_i(i) for i in range(n_valid_batches)] # Create a function that scans the entire test set def test_score(): return [test_score_i(i) for i in range(n_test_batches)] return train_fn, valid_score, test_score def sigmoid_activate(self, Xtest, W, b): # code and compute sigmoid_input = Xtest sigmoid_output = numpy.tanh(numpy.dot(sigmoid_input, W.get_value(borrow=True)) + b.get_value(borrow=True)) return sigmoid_output def softmax_activate(self, Xtest, logLayer): # code and compute softmax_input = Xtest v = numpy.exp( numpy.dot(softmax_input, logLayer.W.get_value(borrow=True)) + logLayer.b.get_value(borrow=True)) softmax_output = v/numpy.sum(v) return softmax_output def predict_functions(self, Xtest): ''' Given a set_x of examples produce a vector y' of predictions by the DBN. ''' tmp = Xtest for L in self.sigmoid_layers: tmp = self.sigmoid_activate( tmp, L.W, L.b ) # finalize with log layer tmp = self.softmax_activate( tmp, self.logLayer ) return tmp
def main(): np.random.seed(seed=1) products = pd.read_csv('../Datafiles/amazon_baby.csv') review_without_punctuation = products['review'].apply( LR.remove_punctuation) products['word_count'] = review_without_punctuation.apply(LR.countWords) products['clear_words'] = review_without_punctuation # print 5 items in the list print review_without_punctuation.head() # remove all rating 3 , they are considered as neutral products = products[products['rating'] != 3] products['sentiment'] = products['rating'].apply(lambda rating: +1 if rating > 3 else -1) mask = np.random.rand(len(products)) < 0.8 train = products[mask] test = products[~mask] print "Training data size", len(train), "Testing data size", len(test) train_reviews = np.array(train['clear_words']) train_labels = np.array(train['sentiment']) vect = CountVectorizer().fit(train_reviews) trained_vectorized = vect.transform(train_reviews) sentiment_model = LogisticRegression() sentiment_model.fit(trained_vectorized, train_labels) print sentiment_model.coef_ print np.sum(sum(sentiment_model.coef_ >= 0)) sample_test_data = test[10:13] print sample_test_data # 1, 1, -1 for sentiment sample_test = vect.transform(sample_test_data['clear_words']) # first test print sentiment_model.predict(sample_test) # result is 1, 1, -1 scores = sentiment_model.decision_function(sample_test) probability = LR.compute_probability(scores) print probability # start to use test data test_vect = vect.transform(test['clear_words']) test_pred = sentiment_model.predict(test_vect) test_scores = sentiment_model.decision_function(test_vect) test_prob = LR.compute_probability(test_scores) test['predictions'] = test_pred test.sort('predictions', ascending=False) print test.head(20)
def __init__(self, numpy_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10): """This class is made to support a variable number of layers. :type numpy_rng: numpy.random.RandomState :param numpy_rng: numpy random number generator used to draw initial weights :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` :type n_ins: int :param n_ins: dimension of the input to the DBN :type hidden_layers_sizes: list of ints :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int :param n_outs: dimension of the output of the network """ self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) assert self.n_layers > 0 if not theano_rng: theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30)) # allocate symbolic variables for the data self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector # of [int] labels # end-snippet-1 # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first # construct the DBN as a deep multilayer perceptron, and when # constructing each sigmoidal layer we also construct an RBM # that shares weights with that layer. During pretraining we # will train these RBMs (which will lead to chainging the # weights of the MLP as well) During finetuning we will finish # training the DBN by doing stochastic gradient descent on the # MLP. for i in range(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden # units of the layer below or the input size if we are on # the first layer if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # the input to this layer is either the activation of the # hidden layer below or the input of the DBN if you are on # the first layer if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=numpy_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) # its arguably a philosophical question... but we are # going to only declare that the parameters of the # sigmoid_layers are parameters of the DBN. The visible # biases in the RBM are parameters of those RBMs, but not # of the DBN. self.params.extend(sigmoid_layer.params) # Construct an RBM that shared weights with this layer rbm_layer = RBM(numpy_rng=numpy_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We now need to add a logistic layer on top of the MLP self.logLayer = LogisticRegression( input=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined as the # negative log likelihood of the logistic regression (output) layer self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logLayer.errors(self.y)
import pandas as pd from sklearn.model_selection import train_test_split from sklearn import preprocessing import matplotlib.pyplot as plt import TwoDimensionPlot ##################################模型1######################################## #导入数据集 pima = pd.read_csv('pima-indians-diabetes.csv') pd0 = pima[pima.columns.tolist()[0:len(pima.columns) - 1]] pd1 = pima['label'] #归一化 pd0 = pd.DataFrame(preprocessing.scale(pd0)) pima = pd.concat([pd0, pd1], axis=1) pima_train, pima_test = train_test_split(pima, test_size=0.3) #定义模型:迭代1000次,学习率默认为0.0001,训练方式随机梯度下降 model_lr1 = LogisticRegression.LogisticRegression(n_iter=1000, gd='sgd') #训练模型 model_lr1.fit(pima_train) print("模型1训练集精确率:", model_lr1.accuracy(pima_train)) print("模型1测试集精确率:", model_lr1.accuracy(pima_test)) print("模型1权重为:", model_lr1.weights[0]) #导入数据集 data = pd.read_csv('test.csv') #划分数据集 data_train, data_test = train_test_split(data, test_size=0.3) #############################模型2############################################# ##定义模型:迭代10000次,学习率为0.01,训练方式默认批量梯度下降,使用L2正则化 model_lr2 = LogisticRegression.LogisticRegression(eta=0.01, n_iter=10000,
def DisplayDigit(sample, label): sample = sample.reshape(settings.WIDTH, settings.HEIGHT) [fig, axs] = plt.subplots(1,1) axs.imshow(sample, cmap='Greys_r') print label ############### main function ################# if __name__ == '__main__': SetupPath() from settings import * from ReadDataSet import * from LogisticRegression import * import settings settings.init() dataset = ReadDataSet() dataset.Read(GetCurrentPath() + '/Data/mnist.pkl') [train_data, train_labels] = dataset.getTrainData() DisplayDigit(train_data[8], train_labels[8]) LR = LogisticRegression(train_data, -1, train_labels) Wt = LR.train(settings.ETA, settings.EPOCH) import cPickle with open(GetCurrentPath() + '/Models/weights.pkl', 'wb') as fp: cPickle.dump(Wt, fp)
# 10-701 Machine Learning, Spring 2011: Homework 3 # Solution code for question 5 of Section 2 import numpy as np import LogisticRegression X_tr = np.loadtxt(open("Data/tr_X.txt", "rb"), delimiter=",") y_tr = np.loadtxt(open("Data/tr_y.txt", "rb"), delimiter=",") X_te = np.loadtxt(open("Data/te_X.txt", "rb"), delimiter=",") y_te = np.loadtxt(open("Data/te_y.txt", "rb"), delimiter=",") ''' # Let's see the digit represented by this vector plt.imshow(X_tr[1000].reshape((16,16))) plt.gray() plt.show() ''' model = LogisticRegression.LogisticRegression() # Pass different values of lambda for solution of 5(b) model.train_model(X_tr, y_tr, X_te, y_te, lamda=1000)
def testTwo(self): z = np.array([4, 5, 6]) expected = np.array([0.9820, 0.9933, 0.9975]) np.testing.assert_almost_equal(LR.sigmoid(z), expected, decimal=4)