def cost_function_regularized(theta, X, y, learningRate): # correct √
theta = np.matrix(theta)
X = np.matrix(X)
y = np.matrix(y)
first = np.multiply(-y, np.log(lr.sigmoid(X * theta.T)))
second = np.multiply((1 - y), np.log(1 - lr.sigmoid(X * theta.T)))
reg = (learningRate /
(2 * len(X))) * np.sum(np.power(theta[:, 1:theta.shape[1]], 2))
return np.sum(first - second) / len(X) + reg
def five_fold_validation(data_set, data_name, eta=None, demo = False):
# split the datasets into fifths
splits = u.five_fold_split(data_set)
errors = []
export = True
# for each fifth of the dataset
for split in splits:
test_set = None
training_set = pd.DataFrame(columns=data_set.columns.values)
# check each fifth
for s in splits:
# if fifth in question
if s == split:
# this fifth is test set
test_set = splits[s]
# all others are training sets
else:
training_set = training_set.append(splits[s], sort=False)
# only export and demonstrate one of the folds
if split != 1:
export = False
else:
export = True
# if eta is supplied, perform Linear Regression
if eta:
model = Logistic_Regression.learn_models(training_set, eta, data_name, export=export)
Logistic_Regression.classify(test_set, model)
# of no eta is supplied, perform Naive Bayes
else:
model = Naive_Bayes.learn(training_set, data_name, export=export)
Naive_Bayes.classify(test_set, model)
# find and append the classification error
err = u.classification_error(test_set)
errors.append(err)
# print results of first split
if demo:
print('Sample Training Data\n', training_set.head())
print('\nWeight Vectors')
for m in model:
print(m, model[m])
print('\nClassified Test Set\n',test_set)
break
# remove Guess column to prevent errors in future fold tests
test_set.drop(['Guess'], axis=1, inplace=True)
# retrn average error
return sum(errors)/len(errors)
def gradient_decent_regularized(theta, X, y, alpha, lamda, vistualize=True):
temp = np.mat(np.zeros(theta.shape)) # temp暂存theta的值
parameters = int(theta.shape[1]) # parameters控制循环次数
cost = np.zeros(iters) # cost用来存放每次迭代后的代价函数
for i in range(iters):
subtraction = lr.sigmoid(X * theta.T) - y
for j in range(parameters):
term = np.multiply(subtraction, X[:, j])
if not j:
temp[0, j] = theta[0, j] - (
(alpha / len(X)) * np.sum(term, axis=0)) # len()返回矩阵的行数
else:
temp[0, j] = np.multiply(
theta[0, j], 1 - alpha * lamda / len(X)) - (
(alpha / len(X)) * np.sum(term, axis=0)
) # len()返回矩阵的行数
theta = temp
cost[i] = cost_function_regularized(X, y, theta, lamda)
if vistualize:
print('这是第{}次循环, 代价函数的值为{:.4f}'.format(i + 1, cost[i]), end=' ')
for k in range(parameters):
print(', Ѳ{:d}为{:.4f}'.format(k, theta[0, k]), end=' ')
print('\n')
for k in range(parameters):
print('{:.4f}, '.format(theta[0, k]), end=' ')
print('\n')
if i != 0 and cost[i] > cost[i - 1]: # 检查梯度下降算法是否正常工作
raise ValueError('The value of cost_function becomes larger')
return cost
def __init__(self,
accuracies_3_1=None,
accuracies_3_3=None,
train_acc_3_3=None,
runtimes=None,
binary_threshold=0):
# stores the accuracies and runtimes of the algorithms on the 4 datasets with indices:
# 0: Adult (continuous and binary)
# 1: Banknote (all continuous)
# 2: Ionosphere (all continuous)
# 3: Breast cancer (all binary)
self.accuracies_3_1 = accuracies_3_1
self.accuracies_3_3 = accuracies_3_3
self.train_acc_3_3 = train_acc_3_3
self.runtimes = runtimes
# 0 by default (i.e. defaults to handling continuous features in naive bayes)
self.binary_threshold = binary_threshold
self.NB = Naive_Bayes()
self.LR = Logistic_Regression()
def __init__(self, n_in, n_hidden_1, n_hidden_2, n_hidden_3, n_out, rng):
#三层隐藏层
self.hidden_1 = hidden_layer(n_in, n_hidden_1, rng)
self.hidden_2 = hidden_layer(n_hidden_1, n_hidden_2, rng)
self.hidden_3 = hidden_layer(n_hidden_2, n_hidden_3, rng)
#一层逻辑回归层
self.logistic = LR.logistic_regression(n_hidden_3, n_out)
#将连接系数、偏差和激活函数置于列表
self.W = [
self.hidden_1.W, self.hidden_2.W, self.hidden_3.W, self.logistic.W
]
self.b = [
self.hidden_1.b, self.hidden_2.b, self.hidden_3.b, self.logistic.b
]
self.activation = [
self.hidden_1.activation, self.hidden_2.activation,
self.hidden_3.activation, 'sigmoid'
]
def gradient_decent_regularized(theta, X, y, learningRate):
theta = np.matrix(theta)
X = np.matrix(X)
y = np.matrix(y)
parameters = int(theta.ravel().shape[1])
grad = np.zeros(parameters)
error = lr.sigmoid(X * theta.T) - y
for i in range(parameters):
term = np.multiply(error, X[:, i])
if (i == 0):
grad[i] = np.sum(term) / len(X)
else:
grad[i] = (np.sum(term) / len(X)) + (
(learningRate / len(X)) * theta[:, i])
return grad
def test(self):
tobeTrained = []
for i in self.datatest:
tobeTrained.append(self.x[i])
#for i in self.generes:
# tobeTrained.append(self.Data[i])
tobeTrained = np.array(tobeTrained)
tobeTrained = tobeTrained.T
print(tobeTrained[:, 0])
y = []
for i in self.Data["rate"]:
if i == "High":
y.append(1)
elif i == "Low":
y.append(-1)
else:
y.append(0)
#del y[0]
X_train, X_test, y_train, y_test = train_test_split(tobeTrained,
y,
test_size=0.20)
print("length of X_train ", np.shape(X_train))
X_trainWithoutPCA = X_train
X_train, X_test = self.dimensionaltyReduction(5, X_train, X_test)
obj1 = LR.Logistic_Regression(X_train, y_train, X_test, y_test)
obj1.FitModel()
obj1.TrainAndTestModel()
obj2 = svm.SVM(X_train, y_train, X_test, y_test)
obj2.FitModel()
obj2.TrainAndTestModel()
testaya = [0, 1.929883, 0, 90, 16]
self.fillMissingTestData(X_trainWithoutPCA, testaya)
def cost_function_regularized(theta, X, y, lamda): # correct √
plus = (lamda / (2 * X.shape[0])) * np.sum(np.power(theta, 2))
return lr.cost_function(X=X, y=y, theta=theta) + plus
) # len()返回矩阵的行数
theta = temp
cost[i] = cost_function_regularized(X, y, theta, lamda)
if vistualize:
print('这是第{}次循环, 代价函数的值为{:.4f}'.format(i + 1, cost[i]), end=' ')
for k in range(parameters):
print(', Ѳ{:d}为{:.4f}'.format(k, theta[0, k]), end=' ')
print('\n')
for k in range(parameters):
print('{:.4f}, '.format(theta[0, k]), end=' ')
print('\n')
if i != 0 and cost[i] > cost[i - 1]: # 检查梯度下降算法是否正常工作
raise ValueError('The value of cost_function becomes larger')
return cost
# print(lr.ensure_alpha(X, y, theta, alpha, iters)) # 确定学习速率
# print(gradient_decent_regularized(X, y, theta, lamda))
# theta_new, cost = gradient_decent_regularized(theta, X, y, alpha, lamda, vistualize=False)
# expected_result = lr.predict(theta_new, X)
expected_result = opt.fmin_tnc(func=cost_function_regularized,
x0=theta,
fprime=gradient_decent_regularized,
args=(X, y, lamda))
print(expected_result)
lr.compare(expected_result, y)
Y_output = np.concatenate((Y_class, Y_outside), axis=0)
return X_output, Y_output
X_train, Y_train, X_validation, Y_validation, X_test, Y_test = loadMNIST(
'/home/felippe/Área de Trabalho/Felippe/Mestrado/'
'Machine_Learning/DataBase/Computer_Vision/MNIST/')
digit = 0
X_train, Y_train = separatingClasses(5400, digit, X_train, Y_train)
X_validation, Y_validation = separatingClasses(540, digit, X_validation,
Y_validation)
X_test, Y_test = separatingClasses(1000, digit, X_test, Y_test)
lr = logi_reg.LogisticRegression()
#X must be in the form (N_X, M)
lr.run((X_train.T) / 255, np.expand_dims(Y_train, axis=1).T)
cl = clc.Classification()
Y_pred = lr.predict((X_validation.T) / 255)
#Finding the best threshold
threshold = np.linspace(0.1, 0.9, 9)
F1_best = -1
for i in threshold:
Y_label_pred = cl.prob2Label(Y_pred, i)
F1 = cl.F1_score(np.expand_dims(Y_validation, axis=1).T, Y_label_pred)
if (F1 > F1_best):
features.replace(features['Parch'], random_var2)
random_var3 = dm.norm_x(features['Fare'], features['Fare'].max(),
features['Fare'].min())
features.replace(features['Fare'], random_var3)
# split_data into training and testing
df_train = dm.split_data(features)
y = df['Survived'].iloc[0:699] #label
x = df_train.values
m = x.shape[0]
bias = np.ones([m, 1])
x = np.hstack((bias, x))
theta = np.zeros(x.shape[1])
y_hat = log_r.activation(np.dot(x, theta))
y = y.values
alpha = 0.005
theta_out = log_r.gradient_descent(x, y, theta, alpha, m, y_hat)
for i in range(10000):
h1 = log_r.activation(np.dot(x, theta_out))
theta_out = log_r.gradient_descent(x, y, theta_out, alpha, m, h1)
loss_out = log_r.loss(x, y, theta_out, m, h1)
if i % 5000 == 0:
print(loss_out)
if i == 9999:
theta_final = theta_out
# Testing the remaining dataset and Prediction
def get_ans():
lr = LR.Logistic_Regression()
lr.SGD_getAns()
from Logistic_Regression import *
import numpy as np
import matplotlib.pyplot as plt
if __name__ == '__main__':
N = 2000
model = Logistic_Regression()
train_x, train_y = model.load_data('./hw3_train.dat')
test_x, test_y = model.load_data('./hw3_test.dat')
Ein, Eout, Ein_S, Eout_S = model.fit(train_x, train_y, test_x, test_y, N)
t = range(N)
plt.style.use('ggplot')
plt.xlabel('$t$')
plt.ylabel('$E_{in}$')
plt.plot(t, Ein, t, Ein_S)
plt.legend(['GD ($\eta=0.01$)', 'SGD ($\eta=0.001$)'])
plt.title('$E_{in}(\mathbf{w}_t)$ as a function of $t$')
plt.savefig('./Ein.pdf', format="pdf")
plt.show()
# Logistic Regression Using Gradient Descent
print("=" * 40)
print("Linear Regression Using Gradient Descent")
data = pd.read_csv("Logistic_Regression_Dataset.csv")
v = len(data.columns)
X = data.iloc[:, 1:v - 1].values
Y = data.iloc[:, v - 1].values
# Splitting Data_Sets
val = len(X) // 3
X_train = X[val:]
X_test = X[:val]
Y_train = Y[val:]
Y_test = Y[:val]
model = Logistic_Regression(1000, 0.021, X_train, Y_train)
model.fit()
model.cost_plot()
predictions = model.predict(X_test)
print("Predictions")
print(predictions)
print("Original Values")
print(Y_test)
print("Accuracy Score : ",
Logistic_Regression.accuracy_metric(Y_test, predictions))
# Linear Regression With L1_L2_Regularization
print("=" * 40)
print("Linear Regression With L1 And L2 Regularization")
data = pd.read_csv("Linear_Regression_Dataset.csv")
v = len(data.columns)
def get_ans():
lr = LR.Logistic_Regression(fileX,fileY)
lr.GD_getAns()
def classify_X(X,y,weight_xy,fold):
kf = KFold(X.shape[0],n_folds=fold,shuffle=True)
current_fold = 1
all_fold_parameter = {}
all_para_inbuilt = {}
for train_index, test_index in kf:
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
#print X_test.shape
#print weight_xy.shape
#exit(0)
for iter in xrange(10000):
layer_0 = X_test
layer_1 = sigmoid_x(np.dot(layer_0, weight))
layer_1_error = layer_1 - y_test
layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1)
synapse_0_derivative = np.dot(layer_0.T, layer_1_delta)
weight_xy -= synapse_0_derivative
y_pridct = map(lambda x:math.ceil(x),layer_1)
#y_pridct = predicted_y(max_h_theta,y)
conf_matrix =LR.make_canfusion_matrix(y_test,y_pridct,y)
#print (y_pridct==y_test).sum()
c_m = confusion_matrix(np.array(y_test),np.array(y_pridct)) # in-built function of confusion matrix.
all_para = LR.parameter_calculation(conf_matrix,y,y_test,y_pridct)
#lda = LogisticRegression()
#lda.fit(X_train, y_train)
#y_pridct_inb = lda.predict(X_test)
mlp = MLPClassifier(hidden_layer_sizes=(50,), max_iter=10, alpha=1e-4,
algorithm='sgd', verbose=10, tol=1e-4, random_state=1,
learning_rate_init=.1)
mlp.fit(X_train, y_train)
print("Training set score: %f" % mlp.score(X_train, y_train))
print("Test set score: %f" % mlp.score(X_test, y_test))
y_pridct_inb = mlp.predict(X_test)
all_parameter = {}
all_parameter.update({'mse':round((np.average((y_pridct_inb-y_test)**2)),4)})
all_parameter.update({'accuracy':round(accuracy_score(y_test,y_pridct_inb),4)})
all_parameter.update({'precision':round(precision_score(y_test,y_pridct_inb),4)})
all_parameter.update({'recall':round(recall_score(y_test,y_pridct_inb),4)})
all_parameter.update({'f-measure':round(f1_score(y_test,y_pridct_inb),4)})
all_fold_parameter.update({current_fold:all_para})
#all_para_inbuilt.update({current_fold:all_parameter})
current_fold+=1
max_accuracy_fold = max(all_fold_parameter,key = lambda k:float(all_fold_parameter[k]['accuracy']))
#max_accuracy_fold_inbuilt = max(all_para_inbuilt,key = lambda k:float(all_para_inbuilt[k]['accuracy']))
print
print "-"*75
print " "*15,"Parameter Evaluation"
print "-"*75
print
LR.print_parameter(all_fold_parameter[max_accuracy_fold])
print
if(len(np.unique(y))==2):
print
print "-"*75
print " "*15," In Built - Parameter Evaluation"
print "-"*75
print
class Evaluation():
def __init__(self,
accuracies_3_1=None,
accuracies_3_3=None,
train_acc_3_3=None,
runtimes=None,
binary_threshold=0):
# stores the accuracies and runtimes of the algorithms on the 4 datasets with indices:
# 0: Adult (continuous and binary)
# 1: Banknote (all continuous)
# 2: Ionosphere (all continuous)
# 3: Breast cancer (all binary)
self.accuracies_3_1 = accuracies_3_1
self.accuracies_3_3 = accuracies_3_3
self.train_acc_3_3 = train_acc_3_3
self.runtimes = runtimes
# 0 by default (i.e. defaults to handling continuous features in naive bayes)
self.binary_threshold = binary_threshold
self.NB = Naive_Bayes()
self.LR = Logistic_Regression()
def fit_and_predict(self, xTrain, yTrain, xTest, yTest, test_type):
if test_type == "lr":
self.LR.fit(xTrain, yTrain)
return self.LR.score(xTrain, yTrain)
else:
self.NB.get_p_features(xTrain, yTrain)
self.NB.get_priors(yTrain)
return self.NB.score(xTest, yTest)
def K_fold_CV(self, X, y, k):
#np.random.seed(45) #setting random seed allows us to replicate results
N, D = X.shape
shuffle_sequence = np.random.permutation(X.shape[0])
X_shuffled = X[shuffle_sequence, :]
y_shuffled = y[shuffle_sequence]
train_size = math.floor(N / k)
results = np.zeros((2, k))
counter = 0
for i in range(k):
X_test = X_shuffled[counter:counter + train_size, :]
y_test = y_shuffled[counter:counter + train_size]
X_train = np.concatenate([
X_shuffled[0:counter, :], X_shuffled[counter + train_size:N, :]
])
y_train = np.concatenate(
[y_shuffled[0:counter], y_shuffled[counter + train_size:N]])
results[0, i] = (self.fit_and_predict(X_train, y_train, X_test,
y_test, "lr"))
results[1, i] = (self.fit_and_predict(X_train, y_train, X_test,
y_test, "nb"))
counter += train_size
# returns the average accuracy over k folds
return np.average(results, axis=1)
# e.g. file_name = "Adult_Scaled.csv"
def separate_X_Y(self, file_name):
df = pd.read_csv(file_name)
if (file_name == "ionosphere90.csv" or file_name == "ionosphere10.csv"
or file_name == "adult_unscaled90.csv"
or file_name == "adult_unscaled10.csv"):
return np.array(df.iloc[:, 2:-1]), np.array(df.iloc[:, -1])
return np.array(df.iloc[:, 1:-1]), np.array(df.iloc[:, -1])
def main(self):
files90 = [
"adult_unscaled90.csv", "banknote90.csv", "ionosphere90.csv",
"Breast90.csv"
]
files10 = [
"adult_unscaled10.csv", "banknote10.csv", "ionosphere10.csv",
"Breast10.csv"
]
acc_3_1 = np.zeros((2, 4))
acc_3_3 = np.zeros((2, 4, 4))
train_acc_3_3 = np.zeros((2, 4, 4))
# task 3-1
for i in range(0, len(files90)):
print("Starting to process file " + files90[i])
X, Y = self.separate_X_Y(files90[i])
X_10, Y_10 = self.separate_X_Y(files10[i])
# set binary thresholds and hyperparameters
# optimal parameters for ionosphere with momentum/l1 regularization:
# vs beta = 0.5
# optimal parameters for breast cancer w momentum/l1 regularizatoin:
# Optimal parameters for banknote w momentum/l1:
if i == 0:
self.LR = Logistic_Regression(lr=0.006,
penalty='none',
max_iter=1750,
random_state=0,
lambdaa=0.0,
beta=0.9)
self.NB.set_bin_thresh(79)
elif i == 1:
# all features are continuous
self.LR = Logistic_Regression(lr=3e-5,
penalty='l1',
max_iter=8000,
random_state=0,
lambdaa=0.5,
beta=0.9)
self.NB.set_bin_thresh(0)
elif i == 2:
# all features are continuous
self.LR = Logistic_Regression(lr=3e-5,
penalty='l1',
max_iter=10000,
random_state=0,
lambdaa=0.5,
beta=0.9)
self.NB.set_bin_thresh(0)
else:
# all features are binary
self.LR = Logistic_Regression(lr=3e-5,
penalty='l1',
max_iter=600,
random_state=0,
lambdaa=0.75,
beta=0.9)
self.NB.set_bin_thresh(len(X[0]))
acc_3_1[:, i] = self.K_fold_CV(X, Y, 5)
N, D = X.shape
shuffle_sequence = np.random.permutation(X.shape[0])
X_shuffled = X[shuffle_sequence, :]
Y_shuffled = Y[shuffle_sequence]
# train model and do CV on 100%, 85%, 70%, 65% of the other data points
percent_train = [1, 0.85, 0.7, 0.65]
for j in range(4):
print("Running " + str(percent_train[j]) + " " + files90[i])
X_train = X_shuffled[0:math.floor(N * percent_train[j]), :]
Y_train = Y_shuffled[0:math.floor(N * percent_train[j])]
train_acc_3_3[:, i, j] = self.K_fold_CV(X_train, Y_train, 5)
acc_3_3[0, i, j] = self.LR.score(X_10, Y_10)
acc_3_3[1, i, j] = self.NB.score(X_10, Y_10)
self.accuracies_3_1 = acc_3_1
self.accuracies_3_3 = acc_3_3
self.train_acc_3_3 = train_acc_3_3
return acc_3_1, acc_3_3
def predict(theta, X):
probability = lr.sigmoid(X * theta.T)
return [1 if x >= 0.5 else 0 for x in probability]
# Data Preprocessing
#data= d.Data_Process('../Data/disk_sample_smart_log_201707.csv')
#faulty=f.Process_Fault_time()
#faulty.data_processing_fault_time(date)#
#data.update_failure_tag('../Data/fault_time.csv')
#data.tag(days) # to predict failure upto next 30 days
# PCA
pca.PCA('../Data/Final.csv')
#Logistic Regression
print("Logistic Regression (Manually) :")
print("\n")
logisticRegression.accuracy_result()
print("\n")
print("Logistic Regression (Using library): ")
print("\n")
lr_library.logistic_Regression_library()
print("\n")
print("Naive Bayes (Manually) :")
print("\n")
naive_bayes.accuracy_result()
print("\n")
print("Naive Bayes (Using library):")
print("\n")
nb_libary.run()
def __init__(self,
dataset,
learn_rate=0.001,
hidden_layers=(5, 2),
output_layer=1,
threshold=0.5,
alpha=0.0001,
momentum=0.9,
shuffle=True,
Epochs=200):
#Set the learning rate
self.learn_rate = learn_rate
# Alpha is the Regularisation term
self.alpha = alpha
# Whether to shuffle the training data on each epoch
self.shuffle = shuffle
# Set the treshold for binary classification
self.threshold = threshold
# If momentum is non-zero make sure a momentum term is included
if momentum > 0:
self.has_momentum = True
self.momentum = momentum
else:
self.has_momentum = False
self.momentum = momentum
# Epochs refer to how many times to iterate over the training data in its entireity
self.epochs = Epochs
# Create the layers and populate each layer with nodes for the network
all_layers = []
k = 0
# Generate each layer to be filled with the correct number of "nodes" as specified by the user
for i in range(0, len(hidden_layers)):
# Only allow non-zero layers to be created
if hidden_layers[i] != 0:
if len(all_layers) == 0:
all_layers.append([])
if i < len(hidden_layers):
#print(hidden_layers[i])
# initialise a standard node for the layer with random initial weights
# Only the length of the dataset is required as the length includes the label
# even though the label isn't used (saves adding + 1 for the padding weight)
layer_node = LR.Logistic_regression(
learning=self.learn_rate,
alpha=self.alpha,
variable_count=(int(len(dataset[0]))),
threshold=threshold,
has_momentum=self.has_momentum,
momentum=self.momentum)
for node_count in range(hidden_layers[i]):
for weight in range(len(layer_node.weights)):
# The random weights are such that
layer_node.weights[weight] = r.normalvariate(
0, 1)
# Deep copy required to that the sub-details are not identical
all_layers[k].append(copy.deepcopy(layer_node))
k += 1
else:
all_layers.append([])
if i < len(hidden_layers):
#print(hidden_layers[i])
# initialise a standard node for the layer with random initial weights
layer_node = LR.Logistic_regression(
learning=self.learn_rate,
alpha=self.alpha,
variable_count=(int(hidden_layers[i - 1]) + 1),
threshold=threshold,
has_momentum=self.has_momentum,
momentum=self.momentum)
for node_count in range(hidden_layers[i]):
for weight in range(len(layer_node.weights)):
# The random weights are such that
layer_node.weights[weight] = r.normalvariate(
0, 1)
# Deep copy required to that the sub-details are not identical
all_layers[k].append(copy.deepcopy(layer_node))
k += 1
# K is used as a substitute for i to account for the zeros in tuples
# It is a quick fix and will be removed if time
# The final lines capture the output layer and makes it of a set size
# Most cases do not require multiple outputs, but this code was designed
# so that it could be extended to a CNN
all_layers.append([])
layer_node = LR.Logistic_regression(
learning=self.learn_rate,
alpha=self.alpha,
variable_count=(int(hidden_layers[-1]) + 1),
threshold=threshold,
has_momentum=self.has_momentum,
momentum=self.momentum)
for node_count in range(output_layer):
for weight in range(len(layer_node.weights)):
layer_node.weights[weight] = r.normalvariate(0, 1)
all_layers[k].append(copy.deepcopy(layer_node))
self.layers = all_layers
def main(self):
files90 = [
"adult_unscaled90.csv", "banknote90.csv", "ionosphere90.csv",
"Breast90.csv"
]
files10 = [
"adult_unscaled10.csv", "banknote10.csv", "ionosphere10.csv",
"Breast10.csv"
]
acc_3_1 = np.zeros((2, 4))
acc_3_3 = np.zeros((2, 4, 4))
train_acc_3_3 = np.zeros((2, 4, 4))
# task 3-1
for i in range(0, len(files90)):
print("Starting to process file " + files90[i])
X, Y = self.separate_X_Y(files90[i])
X_10, Y_10 = self.separate_X_Y(files10[i])
# set binary thresholds and hyperparameters
# optimal parameters for ionosphere with momentum/l1 regularization:
# vs beta = 0.5
# optimal parameters for breast cancer w momentum/l1 regularizatoin:
# Optimal parameters for banknote w momentum/l1:
if i == 0:
self.LR = Logistic_Regression(lr=0.006,
penalty='none',
max_iter=1750,
random_state=0,
lambdaa=0.0,
beta=0.9)
self.NB.set_bin_thresh(79)
elif i == 1:
# all features are continuous
self.LR = Logistic_Regression(lr=3e-5,
penalty='l1',
max_iter=8000,
random_state=0,
lambdaa=0.5,
beta=0.9)
self.NB.set_bin_thresh(0)
elif i == 2:
# all features are continuous
self.LR = Logistic_Regression(lr=3e-5,
penalty='l1',
max_iter=10000,
random_state=0,
lambdaa=0.5,
beta=0.9)
self.NB.set_bin_thresh(0)
else:
# all features are binary
self.LR = Logistic_Regression(lr=3e-5,
penalty='l1',
max_iter=600,
random_state=0,
lambdaa=0.75,
beta=0.9)
self.NB.set_bin_thresh(len(X[0]))
acc_3_1[:, i] = self.K_fold_CV(X, Y, 5)
N, D = X.shape
shuffle_sequence = np.random.permutation(X.shape[0])
X_shuffled = X[shuffle_sequence, :]
Y_shuffled = Y[shuffle_sequence]
# train model and do CV on 100%, 85%, 70%, 65% of the other data points
percent_train = [1, 0.85, 0.7, 0.65]
for j in range(4):
print("Running " + str(percent_train[j]) + " " + files90[i])
X_train = X_shuffled[0:math.floor(N * percent_train[j]), :]
Y_train = Y_shuffled[0:math.floor(N * percent_train[j])]
train_acc_3_3[:, i, j] = self.K_fold_CV(X_train, Y_train, 5)
acc_3_3[0, i, j] = self.LR.score(X_10, Y_10)
acc_3_3[1, i, j] = self.NB.score(X_10, Y_10)
self.accuracies_3_1 = acc_3_1
self.accuracies_3_3 = acc_3_3
self.train_acc_3_3 = train_acc_3_3
return acc_3_1, acc_3_3
# n = number of features in training set. m = Xtr.shape[0] n = Xtr.shape[1] # Generate bias bias = np.zeros((m,1)) bias[:] = 1 # Initialize learning parameters, theta theta = np.zeros(n+1) # Append bias to training set Xtr = np.hstack((bias,Xtr)) # Perform Logistic Regression using Logistic Regression Module. theta_min = LR.optimize(theta,Xtr,ytr) # Calculate errors theta_min = np.reshape(theta_min,(theta_min.size,1)) # Training htr = LR.sigmoid(np.dot(Xtr,theta_min)) Jtr = 1.0/(2.0*m)*(np.dot((htr-ytr).T,htr-ytr)) # Cross-validation Xcv = np.hstack((bias[0:Xcv.shape[0]],Xcv)) hcv = LR.sigmoid(np.dot(Xcv,theta_min)) Jcv = 1.0/(2.0*Xcv.shape[0])*(np.dot((hcv-ycv).T,hcv-ycv)) # Test Xtst = np.hstack((bias[0:Xtst.shape[0]],Xtst))
def get_ans():
lr = LR.Logistic_Regression()
lr.newton_getAns()
def loadData():
train_x = []
train_y = []
fileIn = open('Testdata.txt', 'r', encoding="utf-8-sig").readlines()
for line in fileIn:
lineArr = line.strip().split()
train_x.append([1.0, float(lineArr[0]), float(lineArr[1])])
train_y.append(float(lineArr[2]))
print(len(train_x))
return np.mat(train_x), np.mat(train_y).transpose()
## step 1: load data
print("step 1: load data...")
train_x, train_y = loadData()
test_x = train_x
test_y = train_y
## step 2: training...
print("step 2: training...")
opts = {'alpha': 0.01, 'maxIter': 20, 'optimizeType': 'smoothStocGradDescent'}
optimalWeights = LR.trainLogRegres(train_x, train_y, opts)
## step 3: testing
print("step 3: testing...")
accuracy = LR.testLogRegres(optimalWeights, test_x, test_y)
## step 4: show the result
print("step 4: show the result...")
print("The classify accuracy is: %.3f%%" % (accuracy * 100))
LR.showLogRegres(optimalWeights, train_x, train_y)