def plot_sample(nonreg=True, k = 0):
(X_train_raw, y_train, X_test_raw, y_test) = get_data()
X_train = transform(X_train_raw)
X_test = transform(X_test_raw)
N = X_train.shape[0]
if nonreg is True:
w = linear_regression(X_train, y_train)
else:
w = weight_decay_regression(X_train, y_train, 10.0**k)
def plot_decision_fn(X):
X_trans = transform(X)
return np.sign(np.dot(X_trans, w))
(cont_x, cont_y, cont_z) = decision_boundary_2D(-1, 1, 0.0025, -1, 1, 0.0025,
plot_decision_fn)
print("E_in :", linear_error(X_train, y_train, w))
print("E_out:", linear_error(X_test, y_test, w))
x_plot = X_test_raw[:,0]
y_plot = X_test_raw[:,1]
c = np.where(y_test==1, 'r', 'b')
plt.scatter(x_plot,y_plot, c=c)
plt.contour(cont_x, cont_y, cont_z, [0], colors='g')
plt.xlim([-1, 1])
plt.ylim([-1, 1])
plt.grid()
plt.show()
def one_vs_rest():
print("Solving problems 7-9")
decay = 1
print("decay=", decay)
digits = list(range(10))
(X_train, y_train, X_test, y_test) = get_digit_data()
#(X_train_trans, X_test_trans) = (transform(X_train), transform(X_test))
for digit in digits:
other_digits = [i for i in range(10) if i != digit]
(X_dig_train, y_dig_train) = get_digits(digit, other_digits, X_train, y_train)
(X_dig_test, y_dig_test) = get_digits(digit, other_digits, X_test, y_test)
w_lin = weight_decay_regression(X_dig_train, y_dig_train, decay)
E_in_lin = linear_error(X_dig_train, y_dig_train, w_lin)
E_out_lin = linear_error(X_dig_test, y_dig_test, w_lin)
(X_train_nonlin, X_test_nonlin) = (transform(X_dig_train), transform(X_dig_test))
w_nonlin = weight_decay_regression(X_train_nonlin, y_dig_train, decay)
E_in_nonlin = linear_error(X_train_nonlin, y_dig_train, w_nonlin)
E_out_nonlin = linear_error(X_test_nonlin, y_dig_test, w_nonlin)
print(digit, "vs rest")
print(" E_in_lin: ", E_in_lin)
print(" E_out_lin: ", E_out_lin)
print(" E_in_nonlin: ", E_in_nonlin)
print(" E_out_nonlin: ", E_out_nonlin)
def one_vs_one():
print("Solving problem 10")
digit_1 = 1
digit_2 = 5
print("Comparing digits", digit_1, "and", digit_2)
(X_train, y_train, X_test, y_test) = get_digit_data()
lamba_list = [0.01, 1]
(X_dig_train, y_dig_train) = get_digits(digit_1, [digit_2], X_train, y_train)
(X_dig_test, y_dig_test) = get_digits(digit_1, [digit_2], X_test, y_test)
(X_dig_train, X_dig_test) = (transform(X_dig_train), transform(X_dig_test))
for decay in lamba_list:
print("lambda =", decay)
w = weight_decay_regression(X_dig_train, y_dig_train, decay)
E_in = linear_error(X_dig_train, y_dig_train, w)
E_out = linear_error(X_dig_test, y_dig_test, w)
print(" E_in: ", E_in)
print(" E_out:", E_out)
def answers():
k_list = np.array([-3, -2, -1, 0, 1, 2, 3])
(X_train_raw, y_train, X_test_raw, y_test) = get_data()
X_train = transform(X_train_raw)
X_test = transform(X_test_raw)
assert X_train.shape[0] == y_train.shape[0]
assert X_test.shape[0] == y_test.shape[0]
w_nonreg = linear_regression(X_train, y_train)
E_in_nonreg = linear_error(X_train, y_train, w_nonreg)
E_out_nonreg = linear_error(X_test, y_test, w_nonreg)
print("Number of train points:", X_train.shape[0])
print("Number of test points: ", X_test.shape[0])
print("\nNon-regularized stats: ")
print(" E_in: ", E_in_nonreg)
print(" E_out: ", E_out_nonreg)
print(" w^2sum: ", np.power(w_nonreg, 2).sum())
print("Regularized stats: ")
E_in_reg = np.zeros(len(k_list))
E_out_reg = np.zeros(len(k_list))
for i, k in enumerate(k_list):
decay = 10.0**k
w_reg = weight_decay_regression(X_train, y_train, decay)
E_in_reg[i] = linear_error(X_train, y_train, w_reg)
E_out_reg[i] = linear_error(X_test, y_test, w_reg)
print(" k =", k, "constant =", decay)
print(" E_in: ", E_in_reg[i])
print(" E_out: ", E_out_reg[i])
print(" w^2sum: ", np.power(w_reg, 2).sum())
def test_weight_decay():
k = 3
(X_train_raw, y_train, X_test_raw, y_test) = get_data()
X_train = transform(X_train_raw)
X_test = transform(X_test_raw)
(N, dim) = X_train.shape
w_reg = weight_decay_regression(X_train, y_train, 10.0**k)
w_reg1 = np.dot(np.linalg.inv(np.dot(X_train.T, X_train) +
(10.0**k/N * np.identity(dim))),
np.dot(X_train.T, y_train))
w_reg2 = np.linalg.solve(np.dot(X_train.T, X_train) + (10.0**k * np.identity(dim)),
np.dot(X_train.T, y_train))
print(w_reg - w_reg1)
#print w_reg - w_reg2
print(w_reg)
E_in_reg = linear_error(X_train, y_train, w_reg)
E_out_reg = linear_error(X_test, y_test, w_reg)
print("k =", k, "constant =", 10.0**k/N)
print(" E_in: ", E_in_reg)
print(" E_out: ", E_out_reg)
print(" w^2sum: ", np.power(w_reg, 2).sum())
def train_and_eval(X_train, y_train, X_val, y_val, X_test, y_test):
w = linear_regression(X_train, y_train)
E_val = linear_error(X_val, y_val, w)
E_out = linear_error(X_test, y_test, w)
return E_val, E_out