def test():
figures_directory = 'figures'
os.makedirs(figures_directory, exist_ok=True)
X, y = logistic_regression.setup_data()
#lamb = 0.1
lamb = 0
w0 = np.array([[7.0], [1.5]])
num_iter = 10
alpha = 0.9
colors = ['green', 'darkorchid']
# Solution values
# expected_w_list =
util.plot_objective_contours(X,
y,
lamb,
title='Newton\'n Method vs. Gradient Descent',
colors='gray',
show_labels=False,
new_figure=True,
show_figure=False,
save_filename=None)
gd_w_list = logistic_regression.gradient_descent(X, y, lamb, alpha, w0,
num_iter)
util.plot_optimization_path(gd_w_list,
color=colors[0],
label='Gradient Descent')
actual_w_list = logistic_regression.newtons_method(X, y, lamb, w0,
num_iter)
# expected_w_list = expected_w_lists[alpha]
# for i in range(num_iter+1):
# assert abs(actual_w_list[i][0,0] - expected_w_list[i][0]) < 0.01 , 'Incorrect weight value found for iter={}, w[0]. Expected w={}, found w={}'.format(i, expected_w_list[i], actual_w_list[i])
# assert abs(actual_w_list[i][1,0] - expected_w_list[i][1]) < 0.01 , 'Incorrect weight value found for iter={}, w[1]. Expected w={}, found w={}'.format(i, expected_w_list[i], actual_w_list[i])
util.plot_optimization_path(actual_w_list,
color=colors[1],
label='Newton\'s Method')
plt.xlim(-8, 8)
plt.ylim(-8, 8)
plt.legend(fontsize=util.get_font_size())
filename = '{}/newtons_method.png'.format(figures_directory)
plt.savefig(filename)
plt.show()
test_score = max_score()
test_output = 'PASS\n'
return test_score, test_output
def d_dimensional_comparison(beta_star, num_points, l):
d = len(beta_star) - 1
X_list = [np.random.uniform(0, 256, num_points) for _ in range(d)]
X = np.column_stack(X_list)
X = np.column_stack((np.ones(num_points), X))
distances = np.array([xi.dot(beta_star) for xi in X])
Y = [1 if expit(dist) > random.random() else -1 for dist in distances]
x1_pos, x2_pos = zip(*[xi[1:] for xi, yi in zip(X, Y) if yi == 1])
x1_neg, x2_neg = zip(*[xi[1:] for xi, yi in zip(X, Y) if yi == -1])
plt.scatter(x1_pos, x2_pos, marker='+', color='red')
plt.scatter(x1_neg, x2_neg, marker='o', color='blue')
x2_star = decision(np.column_stack([np.ones(256),
np.arange(256)]), beta_star)
beta_hat = gradient_descent(X,
Y,
l=l,
epsilon=1e-8,
step_size=1e-2,
max_steps=100)
x2_hat = decision(np.column_stack([np.ones(256),
np.arange(256)]), beta_hat)
plt.plot(np.arange(256), x2_star, color='purple')
plt.plot(np.arange(256), x2_hat, color='green')
plt.show()
def d_dimensional_comparison(beta_star, num_points, l):
d = len(beta_star) - 1
X_list = [np.random.uniform(0, 256, num_points) for _ in range(d)]
X = np.column_stack(X_list)
X = np.column_stack((np.ones(num_points), X))
distances = np.array([xi.dot(beta_star) for xi in X])
Y = [1 if expit(dist) > random.random() else -1
for dist in distances]
x1_pos, x2_pos = zip(*[xi[1:] for xi, yi in zip(X, Y) if yi == 1])
x1_neg, x2_neg = zip(*[xi[1:] for xi, yi in zip(X, Y) if yi == -1])
plt.scatter(x1_pos, x2_pos, marker='+', color='red')
plt.scatter(x1_neg, x2_neg, marker='o', color='blue')
x2_star = decision(np.column_stack([np.ones(256), np.arange(256)]),
beta_star)
beta_hat = gradient_descent(X, Y, l=l, epsilon=1e-8, step_size=1e-2,
max_steps=100)
x2_hat = decision(np.column_stack([np.ones(256), np.arange(256)]),
beta_hat)
plt.plot(np.arange(256), x2_star, color='purple')
plt.plot(np.arange(256), x2_hat, color='green')
plt.show()
def test():
figures_directory = 'figures'
os.makedirs(figures_directory, exist_ok=True)
X, y = logistic_regression.setup_data()
lamb = 0.1
w0 = np.array([[7.0], [1.5]])
num_iter = 10
alpha_unit_test_values = [3, 0.9, 0.3]
colors = ['steelblue', 'green', 'orange']
# Solution values
expected_w_lists = {}
expected_w_lists[3] = [(7.000, 1.500), (1.901, -4.303), (4.081, 14.699),
(-0.143, 4.289), (-3.053, -2.910), (0.855, 15.799),
(-2.402, 5.059), (-4.484, -2.243), (-0.143, 16.128),
(-3.100, 5.289), (-4.868, -1.980)]
expected_w_lists[0.9] = [(7.000, 1.500), (5.470, -0.241), (4.090, 0.330),
(2.839, -0.365), (1.887, 0.894), (0.879, -0.472),
(0.828, 2.561), (-0.117, 0.627), (-0.537, 0.175),
(-0.405, 1.664), (-1.060, 0.102)]
expected_w_lists[0.3] = [(7.000, 1.500), (6.490, 0.920), (5.996, 0.431),
(5.516, 0.131), (5.053, 0.034), (4.605, 0.014),
(4.172, 0.015), (3.756, 0.021), (3.357, 0.031),
(2.975, 0.044), (2.612, 0.062)]
util.plot_objective_contours(X,
y,
lamb,
title='Gradient Descent',
colors='gray',
show_labels=False,
new_figure=True,
show_figure=False,
save_filename=None)
for alpha, color in zip(alpha_unit_test_values, colors):
actual_w_list = logistic_regression.gradient_descent(
X, y, lamb, alpha, w0, num_iter)
expected_w_list = expected_w_lists[alpha]
for i in range(num_iter + 1):
assert abs(
actual_w_list[i][0, 0] - expected_w_list[i][0]
) < 0.01, 'Incorrect weight value found for iter={}, w[0]. Expected w={}, found w={}'.format(
i, expected_w_list[i], actual_w_list[i])
assert abs(
actual_w_list[i][1, 0] - expected_w_list[i][1]
) < 0.01, 'Incorrect weight value found for iter={}, w[1]. Expected w={}, found w={}'.format(
i, expected_w_list[i], actual_w_list[i])
util.plot_optimization_path(actual_w_list,
color=color,
label='alpha = {:.1f}'.format(alpha))
plt.xlim(-8, 8)
plt.ylim(-8, 8)
plt.legend(fontsize=util.get_font_size())
filename = '{}/gradient_descent.png'.format(figures_directory)
plt.savefig(filename)
test_score = max_score()
test_output = 'PASS\n'
return test_score, test_output
def test():
figures_directory = 'figures'
os.makedirs(figures_directory, exist_ok=True)
X, y = logistic_regression.setup_data()
lambda_values = [0.1, 2]
alpha_values = [0.5, 0.2]
w0 = np.array([[7.0], [1.5]])
num_iter = 10
colors = ['green', 'darkorchid']
# Solution values
expected_w_list = [(7.000, 1.500), (-0.499, -0.628), (-0.020, 0.777),
(-0.139, 0.180), (-0.122, 0.310), (-0.121, 0.315),
(-0.121, 0.315), (-0.121, 0.315), (-0.121, 0.315),
(-0.121, 0.315), (-0.121, 0.315)]
for lamb, alpha in zip(lambda_values, alpha_values):
util.plot_objective_contours(
X,
y,
lamb,
title='Newton\'s Method vs. Gradient Descent, lambda={}'.format(
lamb),
colors='gray',
show_labels=False,
new_figure=True,
show_figure=False,
save_filename=None)
gd_w_list = logistic_regression.gradient_descent(
X, y, lamb, alpha, w0, num_iter)
util.plot_optimization_path(gd_w_list,
color=colors[0],
label='Gradient Descent')
actual_w_list = logistic_regression.newtons_method(
X, y, lamb, w0, num_iter)
util.plot_optimization_path(actual_w_list,
color=colors[1],
label='Newton\'s Method')
plt.xlim(-8, 8)
plt.ylim(-8, 8)
plt.legend(fontsize=util.get_font_size())
filename = '{}/newtons_method_lambda_{:.1f}.png'.format(
figures_directory, lamb)
filename = filename.replace('.', '_', 1)
plt.savefig(filename)
for i in range(num_iter + 1):
assert abs(
actual_w_list[i][0, 0] - expected_w_list[i][0]
) < 0.01, 'Incorrect weight value found for iter={}, w[0]. Expected w={}, found w={}'.format(
i, expected_w_list[i], actual_w_list[i])
assert abs(
actual_w_list[i][1, 0] - expected_w_list[i][1]
) < 0.01, 'Incorrect weight value found for iter={}, w[1]. Expected w={}, found w={}'.format(
i, expected_w_list[i], actual_w_list[i])
test_score = max_score()
test_output = 'PASS\n'
return test_score, test_output
kit.plot_data_classify(X, y, 'Exam #1 Score', 'Exam #2 Score')
# cost function
one_v = np.reshape(np.ones(m), (m, 1)) # alternative to vstack
X1 = np.concatenate((one_v, X), axis=1) # two columns
initial_theta = np.zeros(
(X.shape[1] + 1, 1)) # Column vector with 1 + # of features (X.shape[1])
# calculate cost function :: ans = 0.693
import logistic_regression as lr
J = lr.compute_cost(initial_theta, X1, y)
print("With theta = [0; 0; 0] ... Cost computed = {:7.3f}".format(J))
# run gradient descent :: ans = [ [-0.1], [-12.0092], [-11.2628] ]
grad = lr.gradient_descent(initial_theta, X1, y)
print("Calculated GD = \n", grad)
# Compute and display cost and gradient with non-zero theta
test_theta = np.array([[-24], [0.2], [0.2]])
J = lr.compute_cost(test_theta, X1, y)
grad = lr.gradient_descent(test_theta, X1, y)
print('Cost at test theta: {:7.3f}'.format(J)) # ans = 0.218
print('Gradient at test theta: \n', grad) # ans = [[0.043], [2.566], [2.647]]
# overlay the decision boundary on the data
# but, first compute the optimized theta for global min :: ans = [[-25.161], [0.206], [0.201]]
theta = lr.optimizer_func(initial_theta, X1, y)
print('Computed theta: ', theta)
theta = np.vstack(theta)