def predict_gradient(X, zx, betas):
# Step 1 - Combine the predication to your other data
X_plus_z = np.append(X, zx, 0)
# Step 2 - Normalize the combined data
X_plus_z_normalized = amf.feature_normalization(X_plus_z)
# Step 3 - Extract the normalized prediction data
z_normalized = np.array([X_plus_z_normalized[-1]])
# Step 4 - Predict the y
zy = predict(amf.extended_matrix(z_normalized), betas[-1])
return zy
def load_data():
# Step 1 - Load Data
Csv_data = np.loadtxt("./A2_datasets_2020/GPUBenchmark.csv",
delimiter=',') # load csv
# Step 2 - Setup X and y
X = Csv_data[:, :-1]
y = Csv_data[:, 6]
# Step 3 - Normalize Data
Xn = amf.feature_normalization(X)
# Step 4 - Return normalized X and labels
return Xn, y
def exerciseA_1_gradient():
print("\nExercise A.1 Gradient")
# Load Data
X, y = load_data()
### Gradient
# Step 1 - Normalize and Extend X
Xe_n = amf.extended_matrix(amf.feature_normalization(X))
# Step 2 - Calculate betas using gradient descent
betas = lirf.gradient_descent(Xe_n, y, alpha=.001, n=1000)
# Step 3 - Calculate cost function for each beta
J_gradient = []
for i, j in enumerate(betas):
J_grad = lirf.cost_function(Xe_n, betas[i], y)
J_gradient.append(J_grad)
# Step 4 - Plot the cost over iterations
fig, ax1 = plt.subplots()
fig.suptitle('Ex A.1 Gradient Descent, alpha = .001', fontsize=14)
ax1.set(xlabel="Number of iterations = " + str(len(betas)),
ylabel="Cost J, min = " + str(round(J_gradient[-1], 3)))
ax1.plot(np.arange(0, len(betas)), J_gradient)
plt.xlim(0, len(betas))
plt.show()
# Step 5 - Predict arbitrary height
# 5a) Place in matrix
heights_to_predict = np.array([[65, 70]])
# 5b) Place in matrix
y_parents_grad = lirf.predict_gradient(X, heights_to_predict, betas)
print("==> The predicted height for a girl with parents (65.70) is:\n",
round(y_parents_grad[0], 2))
def exercise7_1():
# 1- Get X and y from insurance.csv
np_data, headers = load_data()
# 2a - Use simple label encoder instead of one hot encoder...
X, y = preprocess_Label_Encoder(np_data)
X = amf.feature_normalization(X)
# 2b - Transform categorical data (some are binary, some are one hot encoded)
#X, y, headers = preprocess_One_Hot_Encoder(np_data, headers)
# 5 - Init vars for standard linear regression
test_size = 0.2
random_state = 0
alphas = np.linspace(.001,50,50)
cv = 5
# Test Polynomials for all methods except Elastic (not enough memory/processor)
print("method,algorithm,degree,score,params")
for deg in range(1,6):
Xr = amf.extended_matrix_deg(X,deg,False)
# Regression using Train Test Split
regression_train_test_split(Xr,y,test_size,random_state,
LinearRegression,"Linear Regression,"+str(deg)+",")
params = {"alpha":alphas,"max_iter":np.array([10000000])}
# Regression using Cross Val Score
regression_cvs(Xr, y, LinearRegression,cv,"Linear Regression,"+str(deg)+",")
# Regression using Grid Search Cross Validation
regression_grid_search_cvs(Xr, y,Lasso,cv,params,alphas,"Lasso Regression,"+str(deg)+",")
regression_grid_search_cvs(Xr, y,Ridge,cv,params,alphas,"Ridge Regression,"+str(deg)+",")
# For Elastic Net, add array of l1_ratio values to params
params = {"alpha":alphas,"l1_ratio":np.linspace(.1,.99,50),"max_iter":np.array([10000000])}
regression_grid_search_cvs(X, y,ElasticNet,cv,params,alphas,"ElasticNet Regression,"+str(deg)+",")
def exercise1_1():
print("\nExercise 1 - Normal Equation")
# Step 1 - Load Data
Csv_data = np.loadtxt("./A2_datasets_2020/GPUBenchmark.csv",
delimiter=',') # load csv
X = Csv_data[:, :-1]
y = Csv_data[:, 6]
# Step 2 - Normalize Data
Xn = amf.feature_normalization(X)
# Step 3 - Plot data
fig, ax = plt.subplots(2, 3)
fig.suptitle('Ex 1.1, Multivariate Data Sets', fontsize=14)
fig.tight_layout(pad=1.0, rect=[0, 0.03, 1, 0.95])
titles = [
"CudaCores", "BaseClock", "BoostClock", "MemorySpeed", "MemoryConfig",
"MemoryBandwidth", "BenchmarkSpeed"
]
# iterate over columns of Xn by using the Transpose of Xn
i, j = 0, 0
for ind, xi in enumerate(Xn.T):
ax[i][j].scatter(xi, y)
ax[i][j].set_title(titles[ind])
#ax[i][j].set_xlim([xi.min()-1.5, xi.max()+1.5])
j += 1
if j == 3: i, j = 1, 0
plt.show()
# Step 4 - Get extended matrix
Xe = amf.extended_matrix(X)
# Step 5 - Get betas using normal equation
betas = lirf.normal_equation(Xe, y)
# Step 6 - Create prediction matrix
pred = np.array([[2432, 1607, 1683, 8, 8, 256]])
# Step 7 - Make prediction
y_pred = lirf.predict(amf.extended_matrix(pred), betas)[0]
print(
"Predicted benchmark:",
y_pred,
" \tActual benchmark: 114",
)
# Step 9 - What is the cost J(β) when using the β computed by
# the normal equation above?
normal_eq_cost = lirf.cost_function(Xe, betas, y)
print("Cost:", normal_eq_cost)
print("\nExercise 1 - Gradient Descent")
# Gradient - Step 1 - Normalize and Extend X
Xe_n = amf.extended_matrix(amf.feature_normalization(X))
# Step 2 - Calculate betas using gradient descent
alpha, n = .01, 1000
betas = lirf.gradient_descent(Xe_n, y, alpha, n)
# Step 3 - Calculate cost function for each beta
J_gradient = []
for i, j in enumerate(betas):
J_grad = lirf.cost_function(Xe_n, betas[i], y)
J_gradient.append(J_grad)
grad_cost = J_gradient[-1]
print("alpha =", str(alpha), " n =", str(n))
print("Cost:", str(grad_cost))
print(
"Gradient cost within",
str(round(100 * abs(grad_cost - normal_eq_cost) / normal_eq_cost, 5)) +
"% of normal cost -> This is less than 1%!")
# Step XXX - Predict benchmark
y_parents_grad = lirf.predict_gradient(
X, np.array([[2432, 1607, 1683, 8, 8, 256]]), betas)
print("Predicted benchmark:", y_parents_grad[0])
def predict(to_predict, X, b):
all_data = np.append(X, to_predict, 0)
all_data_normalized = amf.feature_normalization(all_data)
to_pred_normalized = np.array([all_data_normalized[-1]])
return sigmoid(amf.extended_matrix(to_pred_normalized), b)[-1]
def exerciseA_1():
print("\nExercise A.1")
# Load Data
global X, y
X, y = load_data()
# A1.1 - Plot Data
fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, sharex=True)
fig.suptitle('Ex A.1, Girl Height in inches', fontsize=14)
ax1.set(xlabel="Mom Height", ylabel="Girl Height")
ax2.set(xlabel="Dad Height")
ax1.scatter(X[:, 0], y, c='#e82d8f', marker='1')
ax2.scatter(X[:, 1], y, c='#40925a', marker='2')
plt.show()
# A1.2 - Compute Extended Matrix
Xe_parents = amf.extended_matrix(X)
print("Extended Matrix of Parent's Heights\n", Xe_parents, "\n")
# A1.3 - Compute Normal Equation and Make a Prediction
Beta_normal_parents = lirf.normal_equation(Xe_parents, y)
y_parents_normal_eq = lirf.predict(
amf.extended_matrix(np.array([[65, 70]])), Beta_normal_parents)
print("==> Prediction of girl height with parental heights of 65,70\n",
y_parents_normal_eq[0], "\n")
# A1.4 - Apply Feature Normalization, plot dataset,
# heights should be centered around 0 with a standard deviation of 1.
X_feature_normalized_heights = amf.feature_normalization(X)
fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, sharex=True)
fig.suptitle('Ex A.1, Girl Height in inches', fontsize=14)
ax1.set(xlabel="Mom Height Normalized", ylabel="Girl Height")
ax2.set(xlabel="Dad Height Normalized")
ax1.scatter(X_feature_normalized_heights[:, 0], y, c='#e82d8f', marker='1')
ax2.scatter(X_feature_normalized_heights[:, 1], y, c='#40925a', marker='2')
plt.show()
# A1.5 - Compute the extended matrix Xe and apply the Normal equation
# on the normalized version of (65.70). The prediction should
# still be 65.42 inches.
Xe_feature_normalized_heights = amf.extended_matrix(
X_feature_normalized_heights)
Beta_normal_parents_normalized = lirf.normal_equation(
Xe_feature_normalized_heights, y)
heights_to_predict = np.array([[65, 70]])
Heights_plus_pred = np.append(X, heights_to_predict, 0)
Normalized_heights_plus_pred = amf.feature_normalization(Heights_plus_pred)
Normalized_heights_to_pred = np.array([Normalized_heights_plus_pred[-1]])
y_parents_pred = lirf.predict(
amf.extended_matrix(Normalized_heights_to_pred),
Beta_normal_parents_normalized)
print(
"==> Prediction of girl height with normalized parental heights of 65,70\n",
y_parents_pred[0], "\n")
# A1.6 - Implement the cost function J(β) = n1 (Xeβ − y)T (Xeβ − y) as a
# function of parameters Xe,y,β. The cost for β from the Normal
# equation should be 4.068.
cost_function_normalized = lirf.cost_function(
Xe_feature_normalized_heights, Beta_normal_parents_normalized, y)
print("==> Cost Function (normalized)\n", cost_function_normalized, "\n")
cost_function = lirf.cost_function(Xe_parents, Beta_normal_parents, y)
print("==> Cost Function not-normalized\n", cost_function, "\n")
def exerciseB_1():
print("\nExercise B - Logistic Regression")
# Ex B.1
# Normalize Data
Xn = amf.feature_normalization(X)
# Plot data
fig, ax1 = plt.subplots(1, 1)
fig.suptitle('Ex B.1, Logistic Regression (normalized data)', fontsize=14)
fig.tight_layout(pad=1.0, rect=[0, 0.03, 1, 0.95])
ax1.scatter(
Xn[y > 0, 0], # plot first col on x
Xn[y > 0, 1], # plot second col on y
c='1', # use the classes (0 or 1) as plot colors
label="Admitted",
s=30,
edgecolors='r' # optional, add a border to the dots
)
ax1.scatter(
Xn[y == 0, 0], # plot first col on x
Xn[y == 0, 1], # plot second col on y
c='0', # use the classes (0 or 1) as plot colors
label="Not Admitted",
s=30,
edgecolors='r' # optional, add a border to the dots
)
ax1.legend(loc='upper right')
plt.show()
# Ex B.2
print("\nB.2, sigmoid", lorf.sigmoid_matrix(np.array([[0, 1], [2, 3]])))
# Ex B.3
print("\nB.3, Xe", amf.extended_matrix(X))
# Ex B.4
beta = np.zeros(Xe_n.shape[1])
lcf = lorf.cost_function(Xe_n, beta, y)
print("\nB.4, logistic cost, beta=[0,0,0]::", lcf,
"\n(solution [0,0,0] / .6931)")
# Ex B.5
global lgd
lgd = lorf.gradient_descent(
amf.extended_matrix(amf.feature_normalization(X)), y, .005, 1)
print("\nB.5, gradient_descent alpha=.005 n=1::beta=", lgd, "cost=",
lorf.cost_function(Xe_n, lgd, y),
"\n(solution B1=[.05,0.141,0.125] / J=.6217)")
# Ex B.6
lgd = lorf.gradient_descent(
amf.extended_matrix(amf.feature_normalization(X)), y, .005, 1000)
print("\nB.6, gradient_descent alpha=.005 n=1000::beta=", lgd, "cost=",
lorf.cost_function(Xe_n, lgd, y),
"\n(solution Bn=[1.686,3.923,3.657] / J=.2035)")
# Plot also the linear decision boundary
# Plot also the logistic decision boundary
Xn = amf.feature_normalization(X)
X1 = Xn[:, 0]
X2 = Xn[:, 1]
#Xe_n2 = amf.mapFeature(Xn[:,0], Xn[:,1], 2)
h = .01 # stepsize in the mesh
x_min, x_max = X1.min() - 0.1, X1.max() + 0.1
y_min, y_max = X2.min() - 0.1, X2.max() + 0.1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h)) # Mesh Grid
x1, x2 = xx.ravel(), yy.ravel() # Turn to two Nx1 arrays
XXe = amf.mapFeature(x1, x2, 2) # Extend matrix for degree 2
lgd2 = lorf.gradient_descent(amf.mapFeature(X1, X2, 2), y, .005, 1000)
p = lorf.sigmoid(XXe, lgd2) # classify mesh ==> probabilities
classes = p > 0.5 # round off probabilities
clz_mesh = classes.reshape(xx.shape) # return to mesh format
cmap_light = ListedColormap(["#FFAAAA", "#AAFFAA", "#AAAAFF"]) # mesh plot
cmap_bold = ListedColormap(["#FF0000", "#00FF00", "#0000FF"]) # colors
plt.figure(2)
plt.pcolormesh(xx, yy, clz_mesh, cmap=cmap_light)
plt.scatter(X1, X2, c=y, marker=".", cmap=cmap_bold)
plt.show()
# Ex B.7
predict = lorf.predict(np.array([[45, 85]]), X, lgd)
# Compute training errors
errors = lorf.get_errors(Xe_n, lgd, y)
print("\nB.7, predict [45 85]::", predict, "training errors::", errors,
"\n(solution predict=.77, errors=11")
"""
import numpy as np
import matplotlib.pyplot as plt
import assignment2_logistic_regression_functions as lorf
import assignment2_matrix_functions as amf
from matplotlib.colors import ListedColormap
# Load Data
Csv_data = np.loadtxt("./A2_datasets_2020/admission.csv",
delimiter=',') # load csv
X = Csv_data[:, 0:2]
y = Csv_data[:, -1]
Xe = amf.extended_matrix(X)
Xe_n = amf.extended_matrix(amf.feature_normalization(X))
def exerciseB_1():
print("\nExercise B - Logistic Regression")
# Ex B.1
# Normalize Data
Xn = amf.feature_normalization(X)
# Plot data
fig, ax1 = plt.subplots(1, 1)
fig.suptitle('Ex B.1, Logistic Regression (normalized data)', fontsize=14)
fig.tight_layout(pad=1.0, rect=[0, 0.03, 1, 0.95])
def exercise3_1():
print("\nExercise 3")
# Ex 3.1 Part A
# import data
data = np.loadtxt("./A2_datasets_2020/breast_cancer.csv",
delimiter=',') # load csv
X = data[:, 0:9]
y = data[:, -1] # benign = 2, malignant = 4
# Ex 3.1 Part B
# Split data -
# @NOTE - Using this method was approved on Slack by TA
test_size = 0.2
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=test_size)
# Ex 3.2
# Modify labels
for i in range(len(y_train)):
y_train[i] = 0 if y_train[i] == 2 else 1
for i in range(len(y_test)):
y_test[i] = 0 if y_test[i] == 2 else 1
# Ex 3.3
X_train_extended_normalized = amf.extended_matrix(
amf.feature_normalization(X_train))
alpha, n = .0001, 10000
betas = lorf.gradient_descent(X_train_extended_normalized,
y_train,
alpha,
n,
get_all_betas=True)
costs = []
for i, beta in enumerate(betas):
costs.append([
i,
lorf.cost_function(X_train_extended_normalized, beta, y_train)
])
# Plot data
fig, ax1 = plt.subplots(1, 1)
fig.suptitle('Ex 3.3, Linear Logistic Regression Cost - α = ' +
str(alpha) + ' n = ' + str(n),
fontsize=14)
fig.tight_layout(pad=1.0, rect=[0, 0.03, 1, 0.95])
c0 = np.array(costs)[:, 0]
c1 = np.array(costs)[:, 1]
ax1.plot(c0, c1)
ax1.set_xlabel("N Iterations")
ax1.set_ylabel("Cost")
plt.show()
#print("\nEx 3.3 - See plot")
# Ex 3.4
# Compute training errors
errors = lorf.get_errors(X_train_extended_normalized, beta, y_train)
correct = len(X_train_extended_normalized) - errors
accuracy = (correct - errors) / len(X_train_extended_normalized)
print("\nEx 3.4")
print("Train:: Errors =", errors, "Correct =", correct, "Accuracy =",
accuracy)
# Ex 3.5
# Compute training errors
X_test_extended_normalized = amf.extended_matrix(
amf.feature_normalization(X_test))
test_errors = lorf.get_errors(X_test_extended_normalized, beta, y_test)
test_correct = len(X_test_extended_normalized) - test_errors
test_accuracy = (test_correct -
test_errors) / len(X_test_extended_normalized)
print("\nEx 3.5")
print("Test:: Errors =", test_errors, "Correct =", test_correct,
"Accuracy =", test_accuracy)