def oneVsAll(X, y, num_labels, lamda):
m, n = X.shape
all_theta = np.zeros((num_labels, n + 1))
X = np.hstack((np.ones((m, 1)), X))
cost_function = lambda p, y: lrCostFunction(p, X, y, lamda)[0]
grad_function = lambda p, y: lrCostFunction(p, X, y, lamda)[1]
for i in range(1, num_labels + 1):
initial_theta = np.zeros(n + 1)
all_theta[i - 1, :] = fmin_cg(cost_function, initial_theta, fprime=grad_function,
args=((y == i).astype(int),), maxiter=100, disp=False)
print('Finish oneVsAll checking number: %d' % i)
return all_theta
def output(part_id):
# Random Test Cases
X = np.column_stack(
(np.ones(20), (np.exp(1) * np.sin(np.linspace(1, 20, 20))),
(np.exp(0.5) * np.cos(np.linspace(1, 20, 20)))))
y = np.sin(X[:, 0] + X[:, 1]) > 0
Xm = np.array([[-1, -1], [-1, -2], [-2, -1], [-2, -2], [1, 1], [1, 2],
[2, 1], [2, 2], [-1, 1], [-1, 2], [-2, 1], [-2, 2], [1, -1],
[1, -2], [-2, -1], [-2, -2]]).reshape((16, 2))
ym = np.array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
4]).reshape(16, 1)
t1 = np.sin(np.array(range(1, 24, 2)).reshape(3, 4).T)
t2 = np.cos(np.array(range(1, 40, 2)).reshape(5, 4).T)
fname = srcs[part_id - 1].rsplit('.', 1)[0]
mod = __import__(fname, fromlist=[fname], level=1)
func = getattr(mod, fname)
if part_id == 1:
J = lrCostFunction(np.array([0.25, 0.5, -0.5]), X, y, 0.1)
grad = gradientFunctionReg(np.array([0.25, 0.5, -0.5]), X, y, 0.1)
return sprintf('%0.5f ', np.hstack((J, grad)).tolist())
elif part_id == 2:
return sprintf('%0.5f ', oneVsAll(Xm, ym, 4, 0.1))
elif part_id == 3:
return sprintf('%0.5f ', predictOneVsAll(t1, Xm))
elif part_id == 4:
return sprintf('%0.5f ', predict(t1, t2, Xm))
def output(part_id):
# Random Test Cases
X = np.column_stack((np.ones(20),
(np.exp(1) * np.sin(np.linspace(1, 20, 20))),
(np.exp(0.5) * np.cos(np.linspace(1, 20, 20)))))
y = np.sin(X[:,0] + X[:,1]) > 0
Xm = np.array([[-1,-1],[-1,-2],[-2,-1],[-2,-2],[1,1],[1,2],[2,1],[2,2],[-1,1],
[-1,2],[-2,1],[-2,2],[1,-1],[1,-2],[-2,-1],[-2,-2]]).reshape((16,2))
ym = np.array([1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4]).reshape(16,1)
t1 = np.sin(np.array(range(1,24,2)).reshape(3,4).T)
t2 = np.cos(np.array(range(1,40,2)).reshape(5,4).T)
fname = srcs[part_id-1].rsplit('.',1)[0]
mod = __import__(fname, fromlist=[fname], level=1)
func = getattr(mod, fname)
if part_id == 1:
J = lrCostFunction(np.array([0.25, 0.5, -0.5]), X, y, 0.1)
grad = gradientFunctionReg(np.array([0.25, 0.5, -0.5]), X, y, 0.1)
return sprintf('%0.5f ', np.hstack((J, grad)).tolist())
elif part_id == 2:
return sprintf('%0.5f ', oneVsAll(Xm, ym, 4, 0.1))
elif part_id == 3:
return sprintf('%0.5f ', predictOneVsAll(t1, Xm))
elif part_id == 4:
return sprintf('%0.5f ', predict(t1, t2, Xm))
def output(partId):
# Random Test Cases
X = np.stack([
np.ones(20),
np.exp(1) * np.sin(np.arange(1, 21)),
np.exp(0.5) * np.cos(np.arange(1, 21))
],
axis=1)
y = (np.sin(X[:, 0] + X[:, 1]) > 0).astype(float)
Xm = np.array([[-1, -1], [-1, -2], [-2, -1], [-2, -2], [1, 1], [1, 2],
[2, 1], [2, 2], [-1, 1], [-1, 2], [-2, 1], [-2, 2], [1, -1],
[1, -2], [-2, -1], [-2, -2]])
ym = np.array([0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3])
t1 = np.sin(np.reshape(np.arange(1, 25, 2), (4, 3), order='F'))
t2 = np.cos(np.reshape(np.arange(1, 41, 2), (4, 5), order='F'))
if partId == '1':
J, grad = lrCostFunction(np.array([0.25, 0.5, -0.5]), X, y, 0.1)
out = formatter('%0.5f ', J)
out += formatter('%0.5f ', grad)
elif partId == '2':
out = formatter('%0.5f ', oneVsAll(Xm, ym, 4, 0.1))
elif partId == '3':
out = formatter('%0.5f ', predictOneVsAll(t1, Xm))
elif partId == '4':
out = formatter('%0.5f ', predict(t1, t2, Xm))
return out
def oneVsAll(X, y, num_labels, lamda):
m, n = X.shape
all_theta = np.zeros((num_labels, n + 1))
X = np.hstack((np.ones((m, 1)), X))
cost_function = lambda p, y: lrCostFunction(p, X, y, lamda)[0]
grad_function = lambda p, y: lrCostFunction(p, X, y, lamda)[1]
for i in range(1, num_labels + 1):
initial_theta = np.zeros(n + 1)
all_theta[i - 1, :] = fmin_cg(cost_function,
initial_theta,
fprime=grad_function,
args=((y == i).astype(int), ),
maxiter=100,
disp=False)
print('Finish oneVsAll checking number: %d' % i)
return all_theta
def oneVsAll(X, y, num_labels, l):
X = np.concatenate((np.ones((X.shape[0], 1)), X), axis=1);
m, n = X.shape
all_theta = np.zeros((num_labels, n))
for x in range(0, num_labels):
initial_theta = np.zeros((n))
#print fmin_cg(lambda t: lrCostFunction(X, y == x, initial_theta, l) )
options = {'full_output': True, 'retall': True}
theta, cost, _, _, _, _, _, allvecs = fmin_bfgs(lambda t: lrCostFunction(X, y == x, t, l), initial_theta, maxiter=50, **options)
all_theta[x, :] = theta
return all_theta
def output(partId):
# Random Test Cases
X = column_stack((ones(20), exp(1) * sin(arange(1, 21, 1)), exp(0.5) * cos(arange(1, 21, 1))))
y = (sin(X[:,0] + X[:,1]) > 0).astype(int)
Xm = array([[-1, -1], [-1, -2], [-2, -1], [-2, -2], [1, 1], [1, 2], [2, 1], [2, 2],
[-1, 1], [-1, 2], [-2, 1], [-2, 2], [1, -1], [1, -2], [-2, -1], [-2, -2]])
ym = array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4])
t1 = sin(arange(1, 25, 2).reshape(4, 3, order='F'))
t2 = cos(arange(1, 41, 2).reshape(4, 5, order='F'))
if partId == '1':
J, grad = lrCostFunction(array([0.25, 0.5, -0.5]), X, y, 0.1)
out = sprintf('%0.5f ', J)
return out + sprintf('%0.5f ', grad)
elif partId == '2':
return sprintf('%0.5f ', oneVsAll(Xm, ym, 4, 0.1))
elif partId == '3':
return sprintf('%0.5f ', predictOneVsAll(t1, Xm))
elif partId == '4':
return sprintf('%0.5f ', predict(t1, t2, Xm))
# =================== 1.2 Visualizing the data ===========================
# Randomly select 100 data points to display
rand_indices = np.random.permutation(m)
sel = X[rand_indices[0:100], :]
plt.figure()
displayData(sel, padding=1)
plt.show()
# ===================== 1.3 Vectorizing logistic regression ==============
theta_t = np.array([-2, -1, 1, 2])
X_t = np.hstack((np.ones(
(5, 1)), np.arange(1, 16).reshape(5, 3, order='F') / 10.0))
y_t = np.array([1, 0, 1, 0, 1])
lambda_t = 3
cost, grad = lrCostFunction(theta_t, X_t, y_t, lambda_t)
print('Cost:', cost)
print('Expected cost: 2.534819')
print('Gradients: \n', grad)
print('Expected gradients: \n [ 0.146561 -0.548558 0.724722 1.398003 ]')
# =================== 1.4 One-vs-all classication ========================
l = 0.1
all_theta = oneVsAll(X, y, num_labels, l)
# ==================== 1.4.1 One-vs-all prediction=========================
pred = predictOneVsAll(all_theta, X)
print('Training Set Accuracy:', np.mean(pred == y) * 100)
%
"""
#% Test case for lrCostFunction
print('\nTesting lrCostFunction() with regularization')
X_t = np.hstack((np.ones((len(y), 1)), X))
y_t = np.eye(10)[y].reshape(-1, 10)
from lrCostFunction import lrCostFunction, gradFind
lambda_t = 3
X_s = np.hstack((np.ones((5, 1)), (np.arange(1, 16).reshape(3, 5)).T / 10))
y_s = np.array([1, 0, 1, 0, 1]).reshape(-1, 1)
theta_s = np.array([-2, -1, 1, 2])[:, np.newaxis]
J = lrCostFunction(theta_s, X_s, y_s, lambda_t)
grad = gradFind(theta_s, X_s, y_s, lambda_t)
print('\nCost:', J)
print('Expected cost: 2.534819\n')
print('Gradients:\n')
print(grad)
print('Expected gradients:')
print(' 0.146561\n -0.548558\n 0.724722\n 1.398003\n')
input('Program paused. Press enter to continue.\n')
## ============ Part 2b: One-vs-All Training ============
print('\nTraining One-vs-All Logistic Regression...\n')
# Not WORKING
rand_indices = np.random.permutation(m)
sel = np.take(X, rand_indices[0:100], 0)
plt, h, display_array = displayData(sel)
plt.show()
# ============ Part 2a: Vectorize Logistic Regression ============
# Test case for lrCostFunction
print('\nTesting lrCostFunction() with regularization')
theta_t = np.array([-2, -1, 1, 2]).reshape(4, 1)
#print(theta_t)
X_t = np.column_stack((np.ones(
(5, 1)), (np.reshape(np.arange(1, 16), (5, 3), order="F") / 10)))
#print(X_t)
y_t = np.array([1, 0, 1, 0, 1]).reshape(5, 1)
lambda_t = 3
J, grad = lrCostFunction(theta_t, X_t, y_t, lambda_t, return_grad=True)
print('\nCost: \n', J)
print('Expected cost: 2.534819\n')
print('Gradients:\n')
print(' \n', grad)
print('Expected gradients:\n')
print(' 0.146561\n -0.548558\n 0.724722\n 1.398003\n')
# ============ Part 2b: One-vs-All Training ============
print('\nTraining One-vs-All Logistic Regression...\n')
_lambda = 0.1
all_theta = oneVsAll(X, y, 10, _lambda)
#print(all_theta.shape)
# ================ Part 3: Predict for One-Vs-All ================
example_width = round(np.sqrt(sel.shape[1]))
displayData.displayData(sel,example_width)
## ============ Part 2a: Vectorize Logistic Regression ============
# In this part of the exercise, you will reuse your logistic regression code from the last exercise.
# You task here is to make sure that your regularized logistic regression implementation is vectorized.
# After that, you will implement one-vs-all classification for the handwritten digit dataset.
# Test case for lrCostFunction
print('Testing lrCostFunction() with regularization')
theta_t = np.array([-2, -1, 1, 2])
X_t = np.c_[(np.ones([5,1]),np.arange(1,16).reshape(3,5).T/10)]
y_t = np.array([[1],[0],[1],[0],[1]]) >= 0.5
lambda_t = 3
[J,grad] = lrCostFunction.lrCostFunction(theta_t, X_t, y_t, lambda_t)
#[J,grad] = lrCostFunction.lrCostFunction(initial_theta, X, y, xlambda)
print('Cost: %.7f\nExpected cost: 2.534819\nGradients:'%J)
for grad_i in grad:
print(grad_i)
print('Expected gradients:\n0.146561\n -0.548558\n 0.724722\n 1.398003')
input('Program paused. Press enter to continue.')
## ============ Part 2b: One-vs-All Training ============
print('Training One-vs-All Logistic Regression...')
xlambda = 0.1
all_theta = oneVsAll.oneVsAll(X, y, num_labels, xlambda)
input('Program paused. Press enter to continue.')
def grad_func(t):
return lrCostFunction(X, y, t, lmda)[1]
def cost_func(t):
return lrCostFunction(X, y, t, lmda)[0]
''' lrCostFunction check '''
theta = np.array([-2, -1, 1, 2])
magic3 = np.array([[8, 1, 6],
[3, 5, 7],
[4, 9, 2]])
X = np.concatenate((np.ones((magic3.shape[0], 1)), magic3), axis=1);
y = np.array([[1], [0], [1]]) >= 0.5
l = 3
try:
assert( np.around( lrCostFunction(X, y, theta, l), decimals=4).tolist() == [7.6832] )
assert( np.around( lrGrad(X, y, theta, l), decimals =5).tolist() == [[0.31722], [-0.12768], [2.64812], [4.23787]] )
except:
sys.exit("Unit test failed")
''' oneVsAll check '''
np.set_printoptions(suppress=True)
np.set_printoptions(precision=6)
X = np.r_[ magic3 , np.sin(range(1, 4))[np.newaxis], np.cos(range(1, 4))[np.newaxis]]
y = np.array([[0, 1, 1, 0, 2]]).T
num_labels = 3
l = 0.1
print oneVsAll(X, y, num_labels, l)
print "\n"
# Test case for lrCostFunction
print('\nTesting lrCostFunction() with regularization')
theta_t = np.array([[-2], [-1], [1], [2]])
data = np.arange(1, 16).reshape(3, 5).T
X_t = np.c_[np.ones((5,1)), data/10]
y_t = (np.array([[1], [0], [1], [0], [1]]) >= 0.5) * 1
lambda_t = 3
J, grad = lrCostFunction(theta_t, X_t, y_t, lambda_t), lrGradient(theta_t, X_t, y_t, lambda_t, flattenResult=False)
print('\nCost: f\n', J)
print('Expected cost: 2.534819\n')
print('Gradients:\n')
print(' f \n', grad)
print('Expected gradients:\n')
print(' 0.146561\n -0.548558\n 0.724722\n 1.398003\n')
print('Program paused. Press enter to continue.\n')
pause()
"""## Part 2b: One-vs-All Training ============"""
print('\nTraining One-vs-All Logistic Regression...\n')
digit dataset."""
# Test case for lrCostFunction
print('\nTesting lrCostFunction() with regularization')
theta_t = np.array([[-2], [-1], [1], [2]])
data = np.arange(1, 16).reshape(3, 5).T
X_t = np.c_[np.ones((5, 1)), data / 10]
y_t = (np.array([[1], [0], [1], [0], [1]]) >= 0.5) * 1
lambda_t = 3
J, grad = lrCostFunction(theta_t, X_t, y_t,
lambda_t), lrGradient(theta_t,
X_t,
y_t,
lambda_t,
flattenResult=False)
print('\nCost: f\n', J)
print('Expected cost: 2.534819\n')
print('Gradients:\n')
print(' f \n', grad)
print('Expected gradients:\n')
print(' 0.146561\n -0.548558\n 0.724722\n 1.398003\n')
print('Program paused. Press enter to continue.\n')
pause()
"""## Part 2b: One-vs-All Training ============"""