def cost_function_regularized(theta, X, y, learningRate): # correct √ theta = np.matrix(theta) X = np.matrix(X) y = np.matrix(y) first = np.multiply(-y, np.log(lr.sigmoid(X * theta.T))) second = np.multiply((1 - y), np.log(1 - lr.sigmoid(X * theta.T))) reg = (learningRate / (2 * len(X))) * np.sum(np.power(theta[:, 1:theta.shape[1]], 2)) return np.sum(first - second) / len(X) + reg
def five_fold_validation(data_set, data_name, eta=None, demo = False): # split the datasets into fifths splits = u.five_fold_split(data_set) errors = [] export = True # for each fifth of the dataset for split in splits: test_set = None training_set = pd.DataFrame(columns=data_set.columns.values) # check each fifth for s in splits: # if fifth in question if s == split: # this fifth is test set test_set = splits[s] # all others are training sets else: training_set = training_set.append(splits[s], sort=False) # only export and demonstrate one of the folds if split != 1: export = False else: export = True # if eta is supplied, perform Linear Regression if eta: model = Logistic_Regression.learn_models(training_set, eta, data_name, export=export) Logistic_Regression.classify(test_set, model) # of no eta is supplied, perform Naive Bayes else: model = Naive_Bayes.learn(training_set, data_name, export=export) Naive_Bayes.classify(test_set, model) # find and append the classification error err = u.classification_error(test_set) errors.append(err) # print results of first split if demo: print('Sample Training Data\n', training_set.head()) print('\nWeight Vectors') for m in model: print(m, model[m]) print('\nClassified Test Set\n',test_set) break # remove Guess column to prevent errors in future fold tests test_set.drop(['Guess'], axis=1, inplace=True) # retrn average error return sum(errors)/len(errors)
def gradient_decent_regularized(theta, X, y, alpha, lamda, vistualize=True): temp = np.mat(np.zeros(theta.shape)) # temp暂存theta的值 parameters = int(theta.shape[1]) # parameters控制循环次数 cost = np.zeros(iters) # cost用来存放每次迭代后的代价函数 for i in range(iters): subtraction = lr.sigmoid(X * theta.T) - y for j in range(parameters): term = np.multiply(subtraction, X[:, j]) if not j: temp[0, j] = theta[0, j] - ( (alpha / len(X)) * np.sum(term, axis=0)) # len()返回矩阵的行数 else: temp[0, j] = np.multiply( theta[0, j], 1 - alpha * lamda / len(X)) - ( (alpha / len(X)) * np.sum(term, axis=0) ) # len()返回矩阵的行数 theta = temp cost[i] = cost_function_regularized(X, y, theta, lamda) if vistualize: print('这是第{}次循环, 代价函数的值为{:.4f}'.format(i + 1, cost[i]), end=' ') for k in range(parameters): print(', Ѳ{:d}为{:.4f}'.format(k, theta[0, k]), end=' ') print('\n') for k in range(parameters): print('{:.4f}, '.format(theta[0, k]), end=' ') print('\n') if i != 0 and cost[i] > cost[i - 1]: # 检查梯度下降算法是否正常工作 raise ValueError('The value of cost_function becomes larger') return cost
def __init__(self, accuracies_3_1=None, accuracies_3_3=None, train_acc_3_3=None, runtimes=None, binary_threshold=0): # stores the accuracies and runtimes of the algorithms on the 4 datasets with indices: # 0: Adult (continuous and binary) # 1: Banknote (all continuous) # 2: Ionosphere (all continuous) # 3: Breast cancer (all binary) self.accuracies_3_1 = accuracies_3_1 self.accuracies_3_3 = accuracies_3_3 self.train_acc_3_3 = train_acc_3_3 self.runtimes = runtimes # 0 by default (i.e. defaults to handling continuous features in naive bayes) self.binary_threshold = binary_threshold self.NB = Naive_Bayes() self.LR = Logistic_Regression()
def __init__(self, n_in, n_hidden_1, n_hidden_2, n_hidden_3, n_out, rng): #三层隐藏层 self.hidden_1 = hidden_layer(n_in, n_hidden_1, rng) self.hidden_2 = hidden_layer(n_hidden_1, n_hidden_2, rng) self.hidden_3 = hidden_layer(n_hidden_2, n_hidden_3, rng) #一层逻辑回归层 self.logistic = LR.logistic_regression(n_hidden_3, n_out) #将连接系数、偏差和激活函数置于列表 self.W = [ self.hidden_1.W, self.hidden_2.W, self.hidden_3.W, self.logistic.W ] self.b = [ self.hidden_1.b, self.hidden_2.b, self.hidden_3.b, self.logistic.b ] self.activation = [ self.hidden_1.activation, self.hidden_2.activation, self.hidden_3.activation, 'sigmoid' ]
def gradient_decent_regularized(theta, X, y, learningRate): theta = np.matrix(theta) X = np.matrix(X) y = np.matrix(y) parameters = int(theta.ravel().shape[1]) grad = np.zeros(parameters) error = lr.sigmoid(X * theta.T) - y for i in range(parameters): term = np.multiply(error, X[:, i]) if (i == 0): grad[i] = np.sum(term) / len(X) else: grad[i] = (np.sum(term) / len(X)) + ( (learningRate / len(X)) * theta[:, i]) return grad
def test(self): tobeTrained = [] for i in self.datatest: tobeTrained.append(self.x[i]) #for i in self.generes: # tobeTrained.append(self.Data[i]) tobeTrained = np.array(tobeTrained) tobeTrained = tobeTrained.T print(tobeTrained[:, 0]) y = [] for i in self.Data["rate"]: if i == "High": y.append(1) elif i == "Low": y.append(-1) else: y.append(0) #del y[0] X_train, X_test, y_train, y_test = train_test_split(tobeTrained, y, test_size=0.20) print("length of X_train ", np.shape(X_train)) X_trainWithoutPCA = X_train X_train, X_test = self.dimensionaltyReduction(5, X_train, X_test) obj1 = LR.Logistic_Regression(X_train, y_train, X_test, y_test) obj1.FitModel() obj1.TrainAndTestModel() obj2 = svm.SVM(X_train, y_train, X_test, y_test) obj2.FitModel() obj2.TrainAndTestModel() testaya = [0, 1.929883, 0, 90, 16] self.fillMissingTestData(X_trainWithoutPCA, testaya)
def cost_function_regularized(theta, X, y, lamda): # correct √ plus = (lamda / (2 * X.shape[0])) * np.sum(np.power(theta, 2)) return lr.cost_function(X=X, y=y, theta=theta) + plus
) # len()返回矩阵的行数 theta = temp cost[i] = cost_function_regularized(X, y, theta, lamda) if vistualize: print('这是第{}次循环, 代价函数的值为{:.4f}'.format(i + 1, cost[i]), end=' ') for k in range(parameters): print(', Ѳ{:d}为{:.4f}'.format(k, theta[0, k]), end=' ') print('\n') for k in range(parameters): print('{:.4f}, '.format(theta[0, k]), end=' ') print('\n') if i != 0 and cost[i] > cost[i - 1]: # 检查梯度下降算法是否正常工作 raise ValueError('The value of cost_function becomes larger') return cost # print(lr.ensure_alpha(X, y, theta, alpha, iters)) # 确定学习速率 # print(gradient_decent_regularized(X, y, theta, lamda)) # theta_new, cost = gradient_decent_regularized(theta, X, y, alpha, lamda, vistualize=False) # expected_result = lr.predict(theta_new, X) expected_result = opt.fmin_tnc(func=cost_function_regularized, x0=theta, fprime=gradient_decent_regularized, args=(X, y, lamda)) print(expected_result) lr.compare(expected_result, y)
Y_output = np.concatenate((Y_class, Y_outside), axis=0) return X_output, Y_output X_train, Y_train, X_validation, Y_validation, X_test, Y_test = loadMNIST( '/home/felippe/Área de Trabalho/Felippe/Mestrado/' 'Machine_Learning/DataBase/Computer_Vision/MNIST/') digit = 0 X_train, Y_train = separatingClasses(5400, digit, X_train, Y_train) X_validation, Y_validation = separatingClasses(540, digit, X_validation, Y_validation) X_test, Y_test = separatingClasses(1000, digit, X_test, Y_test) lr = logi_reg.LogisticRegression() #X must be in the form (N_X, M) lr.run((X_train.T) / 255, np.expand_dims(Y_train, axis=1).T) cl = clc.Classification() Y_pred = lr.predict((X_validation.T) / 255) #Finding the best threshold threshold = np.linspace(0.1, 0.9, 9) F1_best = -1 for i in threshold: Y_label_pred = cl.prob2Label(Y_pred, i) F1 = cl.F1_score(np.expand_dims(Y_validation, axis=1).T, Y_label_pred) if (F1 > F1_best):
features.replace(features['Parch'], random_var2) random_var3 = dm.norm_x(features['Fare'], features['Fare'].max(), features['Fare'].min()) features.replace(features['Fare'], random_var3) # split_data into training and testing df_train = dm.split_data(features) y = df['Survived'].iloc[0:699] #label x = df_train.values m = x.shape[0] bias = np.ones([m, 1]) x = np.hstack((bias, x)) theta = np.zeros(x.shape[1]) y_hat = log_r.activation(np.dot(x, theta)) y = y.values alpha = 0.005 theta_out = log_r.gradient_descent(x, y, theta, alpha, m, y_hat) for i in range(10000): h1 = log_r.activation(np.dot(x, theta_out)) theta_out = log_r.gradient_descent(x, y, theta_out, alpha, m, h1) loss_out = log_r.loss(x, y, theta_out, m, h1) if i % 5000 == 0: print(loss_out) if i == 9999: theta_final = theta_out # Testing the remaining dataset and Prediction
def get_ans(): lr = LR.Logistic_Regression() lr.SGD_getAns()
from Logistic_Regression import * import numpy as np import matplotlib.pyplot as plt if __name__ == '__main__': N = 2000 model = Logistic_Regression() train_x, train_y = model.load_data('./hw3_train.dat') test_x, test_y = model.load_data('./hw3_test.dat') Ein, Eout, Ein_S, Eout_S = model.fit(train_x, train_y, test_x, test_y, N) t = range(N) plt.style.use('ggplot') plt.xlabel('$t$') plt.ylabel('$E_{in}$') plt.plot(t, Ein, t, Ein_S) plt.legend(['GD ($\eta=0.01$)', 'SGD ($\eta=0.001$)']) plt.title('$E_{in}(\mathbf{w}_t)$ as a function of $t$') plt.savefig('./Ein.pdf', format="pdf") plt.show()
# Logistic Regression Using Gradient Descent print("=" * 40) print("Linear Regression Using Gradient Descent") data = pd.read_csv("Logistic_Regression_Dataset.csv") v = len(data.columns) X = data.iloc[:, 1:v - 1].values Y = data.iloc[:, v - 1].values # Splitting Data_Sets val = len(X) // 3 X_train = X[val:] X_test = X[:val] Y_train = Y[val:] Y_test = Y[:val] model = Logistic_Regression(1000, 0.021, X_train, Y_train) model.fit() model.cost_plot() predictions = model.predict(X_test) print("Predictions") print(predictions) print("Original Values") print(Y_test) print("Accuracy Score : ", Logistic_Regression.accuracy_metric(Y_test, predictions)) # Linear Regression With L1_L2_Regularization print("=" * 40) print("Linear Regression With L1 And L2 Regularization") data = pd.read_csv("Linear_Regression_Dataset.csv") v = len(data.columns)
def get_ans(): lr = LR.Logistic_Regression(fileX,fileY) lr.GD_getAns()
def classify_X(X,y,weight_xy,fold): kf = KFold(X.shape[0],n_folds=fold,shuffle=True) current_fold = 1 all_fold_parameter = {} all_para_inbuilt = {} for train_index, test_index in kf: X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] #print X_test.shape #print weight_xy.shape #exit(0) for iter in xrange(10000): layer_0 = X_test layer_1 = sigmoid_x(np.dot(layer_0, weight)) layer_1_error = layer_1 - y_test layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1) synapse_0_derivative = np.dot(layer_0.T, layer_1_delta) weight_xy -= synapse_0_derivative y_pridct = map(lambda x:math.ceil(x),layer_1) #y_pridct = predicted_y(max_h_theta,y) conf_matrix =LR.make_canfusion_matrix(y_test,y_pridct,y) #print (y_pridct==y_test).sum() c_m = confusion_matrix(np.array(y_test),np.array(y_pridct)) # in-built function of confusion matrix. all_para = LR.parameter_calculation(conf_matrix,y,y_test,y_pridct) #lda = LogisticRegression() #lda.fit(X_train, y_train) #y_pridct_inb = lda.predict(X_test) mlp = MLPClassifier(hidden_layer_sizes=(50,), max_iter=10, alpha=1e-4, algorithm='sgd', verbose=10, tol=1e-4, random_state=1, learning_rate_init=.1) mlp.fit(X_train, y_train) print("Training set score: %f" % mlp.score(X_train, y_train)) print("Test set score: %f" % mlp.score(X_test, y_test)) y_pridct_inb = mlp.predict(X_test) all_parameter = {} all_parameter.update({'mse':round((np.average((y_pridct_inb-y_test)**2)),4)}) all_parameter.update({'accuracy':round(accuracy_score(y_test,y_pridct_inb),4)}) all_parameter.update({'precision':round(precision_score(y_test,y_pridct_inb),4)}) all_parameter.update({'recall':round(recall_score(y_test,y_pridct_inb),4)}) all_parameter.update({'f-measure':round(f1_score(y_test,y_pridct_inb),4)}) all_fold_parameter.update({current_fold:all_para}) #all_para_inbuilt.update({current_fold:all_parameter}) current_fold+=1 max_accuracy_fold = max(all_fold_parameter,key = lambda k:float(all_fold_parameter[k]['accuracy'])) #max_accuracy_fold_inbuilt = max(all_para_inbuilt,key = lambda k:float(all_para_inbuilt[k]['accuracy'])) print print "-"*75 print " "*15,"Parameter Evaluation" print "-"*75 print LR.print_parameter(all_fold_parameter[max_accuracy_fold]) print if(len(np.unique(y))==2): print print "-"*75 print " "*15," In Built - Parameter Evaluation" print "-"*75 print
class Evaluation(): def __init__(self, accuracies_3_1=None, accuracies_3_3=None, train_acc_3_3=None, runtimes=None, binary_threshold=0): # stores the accuracies and runtimes of the algorithms on the 4 datasets with indices: # 0: Adult (continuous and binary) # 1: Banknote (all continuous) # 2: Ionosphere (all continuous) # 3: Breast cancer (all binary) self.accuracies_3_1 = accuracies_3_1 self.accuracies_3_3 = accuracies_3_3 self.train_acc_3_3 = train_acc_3_3 self.runtimes = runtimes # 0 by default (i.e. defaults to handling continuous features in naive bayes) self.binary_threshold = binary_threshold self.NB = Naive_Bayes() self.LR = Logistic_Regression() def fit_and_predict(self, xTrain, yTrain, xTest, yTest, test_type): if test_type == "lr": self.LR.fit(xTrain, yTrain) return self.LR.score(xTrain, yTrain) else: self.NB.get_p_features(xTrain, yTrain) self.NB.get_priors(yTrain) return self.NB.score(xTest, yTest) def K_fold_CV(self, X, y, k): #np.random.seed(45) #setting random seed allows us to replicate results N, D = X.shape shuffle_sequence = np.random.permutation(X.shape[0]) X_shuffled = X[shuffle_sequence, :] y_shuffled = y[shuffle_sequence] train_size = math.floor(N / k) results = np.zeros((2, k)) counter = 0 for i in range(k): X_test = X_shuffled[counter:counter + train_size, :] y_test = y_shuffled[counter:counter + train_size] X_train = np.concatenate([ X_shuffled[0:counter, :], X_shuffled[counter + train_size:N, :] ]) y_train = np.concatenate( [y_shuffled[0:counter], y_shuffled[counter + train_size:N]]) results[0, i] = (self.fit_and_predict(X_train, y_train, X_test, y_test, "lr")) results[1, i] = (self.fit_and_predict(X_train, y_train, X_test, y_test, "nb")) counter += train_size # returns the average accuracy over k folds return np.average(results, axis=1) # e.g. file_name = "Adult_Scaled.csv" def separate_X_Y(self, file_name): df = pd.read_csv(file_name) if (file_name == "ionosphere90.csv" or file_name == "ionosphere10.csv" or file_name == "adult_unscaled90.csv" or file_name == "adult_unscaled10.csv"): return np.array(df.iloc[:, 2:-1]), np.array(df.iloc[:, -1]) return np.array(df.iloc[:, 1:-1]), np.array(df.iloc[:, -1]) def main(self): files90 = [ "adult_unscaled90.csv", "banknote90.csv", "ionosphere90.csv", "Breast90.csv" ] files10 = [ "adult_unscaled10.csv", "banknote10.csv", "ionosphere10.csv", "Breast10.csv" ] acc_3_1 = np.zeros((2, 4)) acc_3_3 = np.zeros((2, 4, 4)) train_acc_3_3 = np.zeros((2, 4, 4)) # task 3-1 for i in range(0, len(files90)): print("Starting to process file " + files90[i]) X, Y = self.separate_X_Y(files90[i]) X_10, Y_10 = self.separate_X_Y(files10[i]) # set binary thresholds and hyperparameters # optimal parameters for ionosphere with momentum/l1 regularization: # vs beta = 0.5 # optimal parameters for breast cancer w momentum/l1 regularizatoin: # Optimal parameters for banknote w momentum/l1: if i == 0: self.LR = Logistic_Regression(lr=0.006, penalty='none', max_iter=1750, random_state=0, lambdaa=0.0, beta=0.9) self.NB.set_bin_thresh(79) elif i == 1: # all features are continuous self.LR = Logistic_Regression(lr=3e-5, penalty='l1', max_iter=8000, random_state=0, lambdaa=0.5, beta=0.9) self.NB.set_bin_thresh(0) elif i == 2: # all features are continuous self.LR = Logistic_Regression(lr=3e-5, penalty='l1', max_iter=10000, random_state=0, lambdaa=0.5, beta=0.9) self.NB.set_bin_thresh(0) else: # all features are binary self.LR = Logistic_Regression(lr=3e-5, penalty='l1', max_iter=600, random_state=0, lambdaa=0.75, beta=0.9) self.NB.set_bin_thresh(len(X[0])) acc_3_1[:, i] = self.K_fold_CV(X, Y, 5) N, D = X.shape shuffle_sequence = np.random.permutation(X.shape[0]) X_shuffled = X[shuffle_sequence, :] Y_shuffled = Y[shuffle_sequence] # train model and do CV on 100%, 85%, 70%, 65% of the other data points percent_train = [1, 0.85, 0.7, 0.65] for j in range(4): print("Running " + str(percent_train[j]) + " " + files90[i]) X_train = X_shuffled[0:math.floor(N * percent_train[j]), :] Y_train = Y_shuffled[0:math.floor(N * percent_train[j])] train_acc_3_3[:, i, j] = self.K_fold_CV(X_train, Y_train, 5) acc_3_3[0, i, j] = self.LR.score(X_10, Y_10) acc_3_3[1, i, j] = self.NB.score(X_10, Y_10) self.accuracies_3_1 = acc_3_1 self.accuracies_3_3 = acc_3_3 self.train_acc_3_3 = train_acc_3_3 return acc_3_1, acc_3_3
def predict(theta, X): probability = lr.sigmoid(X * theta.T) return [1 if x >= 0.5 else 0 for x in probability]
# Data Preprocessing #data= d.Data_Process('../Data/disk_sample_smart_log_201707.csv') #faulty=f.Process_Fault_time() #faulty.data_processing_fault_time(date)# #data.update_failure_tag('../Data/fault_time.csv') #data.tag(days) # to predict failure upto next 30 days # PCA pca.PCA('../Data/Final.csv') #Logistic Regression print("Logistic Regression (Manually) :") print("\n") logisticRegression.accuracy_result() print("\n") print("Logistic Regression (Using library): ") print("\n") lr_library.logistic_Regression_library() print("\n") print("Naive Bayes (Manually) :") print("\n") naive_bayes.accuracy_result() print("\n") print("Naive Bayes (Using library):") print("\n") nb_libary.run()
def __init__(self, dataset, learn_rate=0.001, hidden_layers=(5, 2), output_layer=1, threshold=0.5, alpha=0.0001, momentum=0.9, shuffle=True, Epochs=200): #Set the learning rate self.learn_rate = learn_rate # Alpha is the Regularisation term self.alpha = alpha # Whether to shuffle the training data on each epoch self.shuffle = shuffle # Set the treshold for binary classification self.threshold = threshold # If momentum is non-zero make sure a momentum term is included if momentum > 0: self.has_momentum = True self.momentum = momentum else: self.has_momentum = False self.momentum = momentum # Epochs refer to how many times to iterate over the training data in its entireity self.epochs = Epochs # Create the layers and populate each layer with nodes for the network all_layers = [] k = 0 # Generate each layer to be filled with the correct number of "nodes" as specified by the user for i in range(0, len(hidden_layers)): # Only allow non-zero layers to be created if hidden_layers[i] != 0: if len(all_layers) == 0: all_layers.append([]) if i < len(hidden_layers): #print(hidden_layers[i]) # initialise a standard node for the layer with random initial weights # Only the length of the dataset is required as the length includes the label # even though the label isn't used (saves adding + 1 for the padding weight) layer_node = LR.Logistic_regression( learning=self.learn_rate, alpha=self.alpha, variable_count=(int(len(dataset[0]))), threshold=threshold, has_momentum=self.has_momentum, momentum=self.momentum) for node_count in range(hidden_layers[i]): for weight in range(len(layer_node.weights)): # The random weights are such that layer_node.weights[weight] = r.normalvariate( 0, 1) # Deep copy required to that the sub-details are not identical all_layers[k].append(copy.deepcopy(layer_node)) k += 1 else: all_layers.append([]) if i < len(hidden_layers): #print(hidden_layers[i]) # initialise a standard node for the layer with random initial weights layer_node = LR.Logistic_regression( learning=self.learn_rate, alpha=self.alpha, variable_count=(int(hidden_layers[i - 1]) + 1), threshold=threshold, has_momentum=self.has_momentum, momentum=self.momentum) for node_count in range(hidden_layers[i]): for weight in range(len(layer_node.weights)): # The random weights are such that layer_node.weights[weight] = r.normalvariate( 0, 1) # Deep copy required to that the sub-details are not identical all_layers[k].append(copy.deepcopy(layer_node)) k += 1 # K is used as a substitute for i to account for the zeros in tuples # It is a quick fix and will be removed if time # The final lines capture the output layer and makes it of a set size # Most cases do not require multiple outputs, but this code was designed # so that it could be extended to a CNN all_layers.append([]) layer_node = LR.Logistic_regression( learning=self.learn_rate, alpha=self.alpha, variable_count=(int(hidden_layers[-1]) + 1), threshold=threshold, has_momentum=self.has_momentum, momentum=self.momentum) for node_count in range(output_layer): for weight in range(len(layer_node.weights)): layer_node.weights[weight] = r.normalvariate(0, 1) all_layers[k].append(copy.deepcopy(layer_node)) self.layers = all_layers
def main(self): files90 = [ "adult_unscaled90.csv", "banknote90.csv", "ionosphere90.csv", "Breast90.csv" ] files10 = [ "adult_unscaled10.csv", "banknote10.csv", "ionosphere10.csv", "Breast10.csv" ] acc_3_1 = np.zeros((2, 4)) acc_3_3 = np.zeros((2, 4, 4)) train_acc_3_3 = np.zeros((2, 4, 4)) # task 3-1 for i in range(0, len(files90)): print("Starting to process file " + files90[i]) X, Y = self.separate_X_Y(files90[i]) X_10, Y_10 = self.separate_X_Y(files10[i]) # set binary thresholds and hyperparameters # optimal parameters for ionosphere with momentum/l1 regularization: # vs beta = 0.5 # optimal parameters for breast cancer w momentum/l1 regularizatoin: # Optimal parameters for banknote w momentum/l1: if i == 0: self.LR = Logistic_Regression(lr=0.006, penalty='none', max_iter=1750, random_state=0, lambdaa=0.0, beta=0.9) self.NB.set_bin_thresh(79) elif i == 1: # all features are continuous self.LR = Logistic_Regression(lr=3e-5, penalty='l1', max_iter=8000, random_state=0, lambdaa=0.5, beta=0.9) self.NB.set_bin_thresh(0) elif i == 2: # all features are continuous self.LR = Logistic_Regression(lr=3e-5, penalty='l1', max_iter=10000, random_state=0, lambdaa=0.5, beta=0.9) self.NB.set_bin_thresh(0) else: # all features are binary self.LR = Logistic_Regression(lr=3e-5, penalty='l1', max_iter=600, random_state=0, lambdaa=0.75, beta=0.9) self.NB.set_bin_thresh(len(X[0])) acc_3_1[:, i] = self.K_fold_CV(X, Y, 5) N, D = X.shape shuffle_sequence = np.random.permutation(X.shape[0]) X_shuffled = X[shuffle_sequence, :] Y_shuffled = Y[shuffle_sequence] # train model and do CV on 100%, 85%, 70%, 65% of the other data points percent_train = [1, 0.85, 0.7, 0.65] for j in range(4): print("Running " + str(percent_train[j]) + " " + files90[i]) X_train = X_shuffled[0:math.floor(N * percent_train[j]), :] Y_train = Y_shuffled[0:math.floor(N * percent_train[j])] train_acc_3_3[:, i, j] = self.K_fold_CV(X_train, Y_train, 5) acc_3_3[0, i, j] = self.LR.score(X_10, Y_10) acc_3_3[1, i, j] = self.NB.score(X_10, Y_10) self.accuracies_3_1 = acc_3_1 self.accuracies_3_3 = acc_3_3 self.train_acc_3_3 = train_acc_3_3 return acc_3_1, acc_3_3
# n = number of features in training set. m = Xtr.shape[0] n = Xtr.shape[1] # Generate bias bias = np.zeros((m,1)) bias[:] = 1 # Initialize learning parameters, theta theta = np.zeros(n+1) # Append bias to training set Xtr = np.hstack((bias,Xtr)) # Perform Logistic Regression using Logistic Regression Module. theta_min = LR.optimize(theta,Xtr,ytr) # Calculate errors theta_min = np.reshape(theta_min,(theta_min.size,1)) # Training htr = LR.sigmoid(np.dot(Xtr,theta_min)) Jtr = 1.0/(2.0*m)*(np.dot((htr-ytr).T,htr-ytr)) # Cross-validation Xcv = np.hstack((bias[0:Xcv.shape[0]],Xcv)) hcv = LR.sigmoid(np.dot(Xcv,theta_min)) Jcv = 1.0/(2.0*Xcv.shape[0])*(np.dot((hcv-ycv).T,hcv-ycv)) # Test Xtst = np.hstack((bias[0:Xtst.shape[0]],Xtst))
def get_ans(): lr = LR.Logistic_Regression() lr.newton_getAns()
def loadData(): train_x = [] train_y = [] fileIn = open('Testdata.txt', 'r', encoding="utf-8-sig").readlines() for line in fileIn: lineArr = line.strip().split() train_x.append([1.0, float(lineArr[0]), float(lineArr[1])]) train_y.append(float(lineArr[2])) print(len(train_x)) return np.mat(train_x), np.mat(train_y).transpose() ## step 1: load data print("step 1: load data...") train_x, train_y = loadData() test_x = train_x test_y = train_y ## step 2: training... print("step 2: training...") opts = {'alpha': 0.01, 'maxIter': 20, 'optimizeType': 'smoothStocGradDescent'} optimalWeights = LR.trainLogRegres(train_x, train_y, opts) ## step 3: testing print("step 3: testing...") accuracy = LR.testLogRegres(optimalWeights, test_x, test_y) ## step 4: show the result print("step 4: show the result...") print("The classify accuracy is: %.3f%%" % (accuracy * 100)) LR.showLogRegres(optimalWeights, train_x, train_y)