def logistic_h(v, theta_h):
# h(t) = h * logistic(at + b)
# theta_h: [h, a, b]
if len(v.shape) == 2:
v = np.reshape(v, (v.shape[0], ))
h = theta_h[0]
a = theta_h[1]
b = theta_h[2]
lp = logistic(a * v + b)
lm = logistic(-a * v - b)
H = h * lp
# Derivatives
dH = np.empty((v.shape[0], 3))
dH[:, 0] = lp
lp_lm_v = lp * lm * v
dH[:, 1] = h * lp_lm_v
dH[:, 2] = h * lp * lm
return (np.atleast_2d(H).T, dH)
def test_network(test_data, test_label, w, b, n):
misclassified = 0
predict_label = []
for k in range(len(test_data)):
x = test_data[k]
y = test_label[k]
# 2 nodes of input layer, n nodes in hidden layer, 1 node in output layer
a = [0 for i in range(n + 3)]
for i in range(2):
a[i] = x[i]
#calculate nodes in hidden layer
for j in range(2, n + 2):
a[j] = logistic(a[0] * w[0][j] + a[1] * w[1][j] + b[j])[1]
#calcalate output node
sumout = sum([w[j][n + 2] * a[j] for j in range(2, n + 2)]) + b[n + 2]
a[n + 2] = logistic(sumout)[1]
if a[n + 2] != y:
misclassified += 1
predict_label.append(a[n + 2])
print("Neural Network misclassified", misclassified)
printFile("p2.dat", test_data, predict_label)
plotData(2)
return misclassified / len(test_data)
def logistic_h2(v, theta_h):
# h(t) = h * logistic(at + b)
# theta_h: [logit(h), a, b]
if len(v.shape) == 2:
v = np.reshape(v, (v.shape[0], ))
h = logistic(theta_h[0])
a = theta_h[1]
b = theta_h[2]
lp = logistic(a * v + b)
lm = logistic(-a * v - b)
H = h * lp
# Derivatives
dH = np.empty((v.shape[0], 3))
dH[:, 0] = logistic(theta_h[0]) * logistic(-theta_h[0]) * lp
lp_lm_v = lp * lm * v
dH[:, 1] = h * lp_lm_v
dH[:, 2] = h * lp * lm
return (H, dH)
def evaluate(self, v):
# h(t) = h * logistic(at + b)
# theta_h: [logit(h), a, b]
# derived on p. 230 - 232 of DPI notebook
if np.isscalar(v):
v = np.asarray([v])
T = v.shape[0]
if len(v.shape) == 2:
v = np.reshape(v, (T, ))
(h, a, b) = (logistic(self.hazard_params[0]), self.hazard_params[1],
self.hazard_params[2])
logmh = self.loglogistic(-self.hazard_params[0])
logh = self.loglogistic(self.hazard_params[0])
logisticNeg = logistic(-a * v - b)
logH = self.loglogistic(a * v + b) + logh
logmH = self.logsumexp(-a * v - b, logmh) + self.loglogistic(a * v + b)
if len(logH.shape) == 1:
logH = np.atleast_2d(logH).T
if len(logmH.shape) == 1:
logmH = np.atleast_2d(logmH).T
# Derivatives
dlogH = np.zeros((T, 3))
dlogH[:, 0] = logistic(-self.hazard_params[0])
dlogH[:, 1] = logisticNeg * v
dlogH[:, 2] = logisticNeg
dlogmH = np.zeros((T, 3))
dlogmH[:, 0] = self.dlogsumexp(-a * v - b, 0, logmh, -h)
dlogmH[:,
1] = self.dlogsumexp(-a * v - b, -v, logmh, 0) + v * logisticNeg
dlogmH[:, 2] = self.dlogsumexp(-a * v - b, -1, logmh, 0) + logisticNeg
assert logH.shape == (T, 1)
assert logmH.shape == (T, 1)
assert dlogH.shape == (T, 3)
assert dlogmH.shape == (T, 3)
return (logH, logmH, dlogH, dlogmH)
def run_logistic_regression():
#train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.99,
'weight_regularization': 0.1,
'num_iterations': 200
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.normal(0, math.sqrt(0.01),train_inputs.shape[1]+1).reshape(-1, 1)
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
valid_set = []
log_likelihood = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
log_likelihood.append(f)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
valid_set.append(frac_correct_valid)
# print some stats
print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t+1, f / N, cross_entropy_train, frac_correct_train*100,
cross_entropy_valid, frac_correct_valid*100)
plt.plot(range(hyperparameters['num_iterations']), log_likelihood)
plt.plot(range(hyperparameters['num_iterations']), valid_set)
plt.axis([0, 140, -10, 140])
#splt.axis([0, 30, -1, 30])
plt.show()
def evaluate(self,v):
# h(t) = h * logistic(at + b)
# theta_h: [logit(h), a, b]
# derived on p. 230 - 232 of DPI notebook
if np.isscalar(v):
v = np.asarray([v])
T = v.shape[0]
if len(v.shape) == 2:
v = np.reshape(v, (T, ))
(h, a, b) = (logistic(self.hazard_params[0]), self.hazard_params[1], self.hazard_params[2])
logmh = self.loglogistic(-self.hazard_params[0])
logh = self.loglogistic(self.hazard_params[0])
logisticNeg = logistic(-a*v - b)
logH = self.loglogistic(a * v + b) + logh
logmH = self.logsumexp(-a * v - b, logmh) + self.loglogistic(a * v + b)
if len(logH.shape) == 1:
logH = np.atleast_2d(logH).T
if len(logmH.shape) == 1:
logmH = np.atleast_2d(logmH).T
# Derivatives
dlogH = np.zeros((T, 3))
dlogH[:,0] = logistic(-self.hazard_params[0])
dlogH[:,1] = logisticNeg*v
dlogH[:,2] = logisticNeg
dlogmH = np.zeros((T, 3))
dlogmH[:,0] = self.dlogsumexp(-a * v - b, 0, logmh, -h)
dlogmH[:,1] = self.dlogsumexp(-a * v - b, -v, logmh, 0) + v*logisticNeg
dlogmH[:,2] = self.dlogsumexp(-a * v - b, -1, logmh, 0) + logisticNeg
assert logH.shape == (T, 1)
assert logmH.shape == (T, 1)
assert dlogH.shape == (T, 3)
assert dlogmH.shape == (T, 3)
return (logH, logmH, dlogH, dlogmH)
def run_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': ...,
'weight_regularization': ...,
'num_iterations': ...
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = ...
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(
weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
def run_logistic_regression(hyperparameters):
# TODO specify training data
train_inputs, train_targets = load_train()
valid_inputs, valid_targets = load_valid()
# N is number of examples; M is the number of features per example.
N, M = train_inputs.shape
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.randn(M + 1, 1) * 0.01
# weights = np.random.normal(0, 0.5, M + 1).reshape(-1, 1)
# weights = np.ones((M + 1, 1)) * 0.0
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
logging = np.zeros((hyperparameters['num_iterations'], 5))
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t + 1, f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100)
logging[t] = [
f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100
]
return logging
def run_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': ...,
'weight_regularization': ...,
'num_iterations': ...
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = ...
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
def run_logistic_regression():
train_inputs, train_targets = load_train()
# train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
hyperparameters = {
"learning_rate": 0.12,
"weight_regularization": 0.,
"num_iterations": 1000
}
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
# run_check_grad(hyperparameters)
print('HYPERPARAMETER: ', hyperparameters['learning_rate'])
alpha = hyperparameters['learning_rate']
weights = np.zeros((M + 1, 1))
f = df = y = None
training_results = []
valid_results = []
for i in range(hyperparameters['num_iterations']):
f, df, y = logistic(weights, train_inputs, train_targets, hyperparameters)
weights = np.subtract(weights, alpha*df)
training_results.append(f)
valid_res = evaluate(valid_targets,
logistic_predict(weights, valid_inputs))
valid_results.append(valid_res[0])
plot_data([i for i in range(1, hyperparameters['num_iterations'] + 1)],
training_results, valid_results)
result = evaluate(train_targets, logistic_predict(weights, train_inputs))
print("MNIST Train Cross Entropy: ", result[0])
print("MNIST Train Classification Rate: ", result[1])
result = evaluate(valid_targets, logistic_predict(weights, valid_inputs))
print("Validation Data on MNIST Train Cross Entropy: ", result[0])
print("Validation Data on MNIST Train Classification Rate: ",
result[1])
result = evaluate(test_targets, logistic_predict(weights, test_inputs))
print("Test Data on MNIST Train Cross Entropy: ", result[0])
print("Test Data on MNIST Train Classification Rate: ",
result[1])
def run_logistic_regression():
# train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
hyperparameters = {
"learning_rate": 0.001,
"weight_regularization": 0.,
"num_iterations": 4000
}
weights = np.zeros((M + 1, 1))
run_check_grad(hyperparameters)
train_entropy = []
validate_entropy = []
for t in range(hyperparameters["num_iterations"]):
f, df, y = logistic(weights, train_inputs, train_targets, hyperparameters)
weights = np.subtract(weights, hyperparameters['learning_rate'] * df)
train_entropy.append(f)
validate_ce = evaluate(valid_targets, logistic_predict(weights, valid_inputs))[0]
validate_entropy.append(validate_ce)
plt.figure()
plt.plot(range(hyperparameters['num_iterations']), train_entropy, label='Training Cross Entropy')
plt.plot(range(hyperparameters['num_iterations']), validate_entropy, label='Validate Cross Entropy')
plt.title('mnist small, learning rate = {}, {} iterations, plot of cross entropy'.format(hyperparameters['learning_rate'], hyperparameters['num_iterations']))
plt.xlabel('Number of Iterations')
plt.ylabel('Cross Entropy')
plt.legend()
plt.show()
print('Learning: {}, number of iterations: {}'.format(hyperparameters['learning_rate'], hyperparameters['num_iterations']))
train_results = evaluate(train_targets, logistic_predict(weights, train_inputs))
print("MNIST small training cross entropy: {}".format(train_results[0]))
print("MNIST small training classification error: {}".format(1 - train_results[1]))
validate_result = evaluate(valid_targets, logistic_predict(weights, valid_inputs))
print("MNIST small validate cross entropy: {}".format(validate_result[0]))
print("MNIST validate classification error: {}".format(1 - validate_result[1]))
test_result = evaluate(test_targets, logistic_predict(weights, test_inputs))
print("MNIST small test cross entropy: {}".format(test_result[0]))
print("MNIST small test classification error: {}".format(1 - test_result[1]))
def run_check_grad(hyperparameters):
""" Performs gradient check on logistic function.
:return: None
"""
# This creates small random data with 20 examples and
# 10 dimensions and checks the gradient on that data.
num_examples = 20
num_dimensions = 10
weights = np.random.randn(num_dimensions + 1, 1)
data = np.random.randn(num_examples, num_dimensions)
targets = np.random.rand(num_examples, 1)
y, dy, = logistic(weights, data, targets, hyperparameters)[:2]
diff = check_grad(logistic, weights, 0.001, data, targets, hyperparameters)
print("diff =", diff)
def run_logistic_regression():
train_inputs, train_targets = load_train()
train_inputs_s, train_targets_s = load_train_small()
#train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.1,
'iterations': 1000,
'iterations_s': 1000
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = 0.1*np.random.randn(M+1,1)
weights_s = 0.1* np.random.randn(M+1,1)
# weights = 0.1*np.random.random_sample((M+1,1))
# weights_s = 0.1*np.random.random_sample((M+1,1))
# plotweight(weights)
# plotweight(weights_s)
ce_train = np.zeros(hyperparameters['iterations'])
ce_valid = np.zeros(hyperparameters['iterations'])
ce_train_s = np.zeros(hyperparameters['iterations_s'])
ce_valid_s = np.zeros(hyperparameters['iterations_s'])
fr_train_s = np.zeros(hyperparameters['iterations_s'])
fr_valid_s = np.zeros(hyperparameters['iterations_s'])
fr_train = np.zeros(hyperparameters['iterations_s'])
fr_valid = np.zeros(hyperparameters['iterations_s'])
it = np.zeros(hyperparameters['iterations'])
it_s = np.zeros(hyperparameters['iterations_s'])
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
# first let's do small set
for t in xrange(hyperparameters['iterations_s']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f_s, df_s, predictions_s = logistic(weights_s, train_inputs_s, train_targets_s, hyperparameters)
# Evaluate the prediction.
cross_entropy_train_s, frac_correct_train_s = evaluate(train_targets_s, predictions_s)
if np.isnan(f_s) or np.isinf(f_s):
raise ValueError("nan/inf error")
# update parameters
weights_s = weights_s - hyperparameters['learning_rate'] * df_s / N
# Make a prediction on the valid_inputs.
predictions_valid_s = logistic_predict(weights_s, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid_s, frac_correct_valid_s = evaluate(valid_targets, predictions_valid_s)
ce_train_s[t] = cross_entropy_train_s
ce_valid_s[t] = cross_entropy_valid_s
fr_train_s[t] = frac_correct_train_s
fr_valid_s[t] = frac_correct_valid_s
it_s[t] = t
# print some stats only for the last round
if t == hyperparameters['iterations']-1:
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f_s / N),
float(cross_entropy_train_s),
float(frac_correct_train_s*100),
float(cross_entropy_valid_s),
float(frac_correct_valid_s*100))
print '---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------now start with the bigger dataset------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------'
# Begin learning with gradient descent
# Now let's Rock n Roll a bigger dataset
for t in xrange(hyperparameters['iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
ce_train[t] = cross_entropy_train
ce_valid[t] = cross_entropy_valid
fr_train[t] = frac_correct_train
fr_valid[t] = frac_correct_valid
it[t] = t
# print some stats only for the last round
if t == hyperparameters['iterations']-1:
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
# plotweight(weights)
#let's start trying test dataset
pre_test = logistic_predict(weights, test_inputs)
pre_test_s = logistic_predict(weights_s, test_inputs)
ce_test, fc_test = evaluate(test_targets, pre_test)
ce_test_s, fc_test_s = evaluate(test_targets, pre_test_s)
# print some test stats
stat_msg = " test CE:{:.6f} "
stat_msg += "test FRAC:{:2.2f} test Error:{:2.2f} test_s CE:{:.6f} test_s FRAC:{:2.2f} test_s Error:{:2.2f}"
print stat_msg.format(
float(ce_test),
float(fc_test*100),
float((1-fc_test)*100),
float(ce_test_s),
float(fc_test_s*100),
float((1-fc_test_s)*100))
#after training see the distribution of weights
# plotweight(weights)
#Below is we can see what's going on for this hyperparameter
plt.figure(figsize=(8,7),dpi=98)
ax1 = plt.subplot(221)
ax2 = plt.subplot(222)
ax3 = plt.subplot(223)
ax4 = plt.subplot(224)
ax1.set_title('mnist_train')
ax1.plot(it, ce_train, marker='o', label='Training Set')
ax1.plot(it, ce_valid, marker='x', label='Validation Set')
legend = plt.legend()
ax1.set_ylabel('Cross Entropy')
ax1.legend(loc='upper right')
ax1.grid()
ax2.set_title('mnist_train_small')
ax2.plot(it_s, ce_train_s, marker='o', label='Training Set')
ax2.plot(it_s, ce_valid_s, marker='x', label='Validation Set')
legend = plt.legend()
ax2.legend(loc='upper right')
ax2.set_ylabel('Cross Entropy')
ax2.grid()
ax3.set_title('mnist_train')
ax3.plot(it, fr_train, marker='o', label='Training Set')
ax3.plot(it, fr_valid, marker='x', label='Validation Set')
ax3.set_ylabel('correction rate')
ax3.grid()
ax4.set_title('mnist_train_small')
ax4.plot(it, fr_train_s, marker='o', label='Training Set')
ax4.plot(it, fr_valid_s, marker='x', label='Validation Set')
ax4.set_ylabel('correction rate')
ax4.grid()
plt.show()
#Below two images to answer question 2.2. Plot cross entropy
plt.figure(1)
plt.title('mnist_train')
plt.plot(it, ce_train, 'r-', label='Training Set')
plt.plot(it, ce_valid, 'x', label='Validation Set')
legend = plt.legend()
plt.xlabel('iter')
plt.ylabel('Cross Entropy')
plt.grid()
plt.figure(2)
plt.title('mnist_train_small')
plt.plot(it, ce_train_s, 'r-', label='Training Set')
plt.plot(it, ce_valid_s, 'x', label='Validation Set')
legend = plt.legend()
plt.xlabel('iter')
plt.ylabel('Cross Entropy')
plt.grid()
plt.show()
def run_logistic_regression():
#train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
#####################################################################
# TODO: #
# Set the hyperparameters for the learning rate, the number #
# of iterations, and the way in which you initialize the weights. #
#####################################################################
hyperparameters = {
"learning_rate": 0.005,
"weight_regularization": 0.,
"num_iterations": 400
}
weights = [[0.]] * (M + 1)
#####################################################################
# END OF YOUR CODE #
#####################################################################
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
#####################################################################
# TODO: #
# Modify this section to perform gradient descent, create plots, #
# and compute test error. #
#####################################################################
print(
"\n=============================================================================\n"
)
print("The step size is {}.".format(hyperparameters["learning_rate"]))
print("The number of iterations is: {}.\n".format(
hyperparameters["num_iterations"]))
ce_list_train = []
ce_list_valid = []
for t in range(hyperparameters["num_iterations"]):
f_train, df_train, y_train = logistic(weights, train_inputs,
train_targets, hyperparameters)
ce_train, frac_correct_train = evaluate(train_targets, y_train)
ce_list_train.append(ce_train)
f_valid, df_valid, y_valid = logistic(weights, valid_inputs,
valid_targets, hyperparameters)
ce_valid, frac_correct_valid = evaluate(valid_targets, y_valid)
ce_list_valid.append(ce_valid)
weights = weights - hyperparameters["learning_rate"] * df_train
ce_train, frac_correct_train = evaluate(train_targets, y_train)
ce_list_train.append(ce_train)
print(
"For the training dataset, the averaged cross entropy is {} and the classification error is {}."
.format(ce_train, 1 - frac_correct_train))
f_valid, df_valid, y_valid = logistic(weights, valid_inputs, valid_targets,
hyperparameters)
ce_valid, frac_correct_valid = evaluate(valid_targets, y_valid)
ce_list_valid.append(ce_valid)
print(
"For the validation dataset, the averaged cross entropy is {} and the classification error is {}."
.format(ce_valid, 1 - frac_correct_valid))
f_test, df_test, y_test = logistic(weights, test_inputs, test_targets,
hyperparameters)
ce_test, frac_correct_test = evaluate(test_targets, y_test)
print(
"For the test dataset, the averaged cross entropy is {} and the classification error is {}.\n"
.format(ce_test, 1 - frac_correct_test))
plt.plot(range(hyperparameters["num_iterations"] + 1),
ce_list_train,
label="Training Dataset")
plt.plot(range(hyperparameters["num_iterations"] + 1),
ce_list_valid,
label="Validation Dataset")
plt.legend(loc="upper right")
plt.xlabel("iteration")
plt.ylabel("Cross Entropy")
plt.show()
print(
"=============================================================================\n\n"
)
def run_logistic_regression():
train_inputs, train_targets = load_train()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.001,
# 'weight_regularization': 0.6,
'num_iterations': 4700
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.randn(M+1, 1)
weights *= 0.01
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
return
y_cross_entropy_train = []
y_cross_entropy_valid = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
# plot for 2.2 CE for train and valid
y_cross_entropy_train.append(cross_entropy_train)
y_cross_entropy_valid.append(cross_entropy_valid)
x_iteration = range(1, hyperparameters['num_iterations'] + 1)
plt.plot(x_iteration, y_cross_entropy_train)
plt.plot(x_iteration, y_cross_entropy_valid)
plt.legend(['train', 'valid'], loc='upper right')
plt.xlabel('iteration')
plt.ylabel('cross entropy')
plt.title('train')
plt.show()
def run_logistic_regression():
train_inputs, train_targets = load_train()
train_inputs_s, train_targets_s = load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.12,
'weight_regularization': 1,
'num_iterations': 800,
'num_iterations_s': 800
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = -0.1+0.2*np.random.random_sample((M+1,1))
weights_s = -0.1+0.2*np.random.random_sample((M+1,1))
ce_train = np.zeros(hyperparameters['num_iterations'])
ce_valid = np.zeros(hyperparameters['num_iterations'])
ce_train_s = np.zeros(hyperparameters['num_iterations_s'])
ce_valid_s = np.zeros(hyperparameters['num_iterations_s'])
it = np.zeros(hyperparameters['num_iterations'])
it_s = np.zeros(hyperparameters['num_iterations_s'])
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations_s']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f_s, df_s, predictions_s = logistic(weights_s, train_inputs_s, train_targets_s, hyperparameters)
# Evaluate the prediction.
cross_entropy_train_s, frac_correct_train_s = evaluate(train_targets_s, predictions_s)
if np.isnan(f_s) or np.isinf(f_s):
raise ValueError("nan/inf error")
# update parameters
weights_s = weights_s - hyperparameters['learning_rate'] * df_s / N
# Make a prediction on the valid_inputs.
predictions_valid_s = logistic_predict(weights_s, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid_s, frac_correct_valid_s = evaluate(valid_targets, predictions_valid_s)
ce_train_s[t] = cross_entropy_train_s
ce_valid_s[t] = cross_entropy_valid_s
it_s[t] = t
# print some stats
if t % 100 == 99:
print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f} "
).format(
t+1, f_s / N, cross_entropy_train_s, frac_correct_train_s*100,
cross_entropy_valid_s, frac_correct_valid_s*100)
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
ce_train[t] = cross_entropy_train
ce_valid[t] = cross_entropy_valid
it[t] = t
# print some stats
if t % 100 == 99:
print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f} "
).format(
t+1, f / N, cross_entropy_train, frac_correct_train*100,
cross_entropy_valid, frac_correct_valid*100)
predictions_test = logistic_predict(weights, test_inputs)
predictions_test_s = logistic_predict(weights_s, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets, predictions_test)
cross_entropy_test_s, frac_correct_test_s = evaluate(test_targets, predictions_test_s)
print ("TEST CE:{:.6f} TEST FRAC:{:2.2f} TEST"
"S CE:{:.6f} TEST S FRAC:{:2.2f}").format(
cross_entropy_test, frac_correct_test*100,
cross_entropy_test_s, frac_correct_test_s*100)
plt.figure(1)
plt.title('Logistic Regression on mnist_train')
plt.plot(it, ce_train, marker='o', label='Training Set')
plt.plot(it, ce_valid, marker='x', label='Validation Set')
legend = plt.legend()
plt.xlabel('k')
plt.ylabel('Cross Entropy')
plt.figure(2)
plt.title('Logistic Regression on mnist_train_small')
plt.plot(it_s, ce_train_s, marker='o', label='Training Set')
plt.plot(it_s, ce_valid_s, marker='x', label='Validation Set')
legend = plt.legend()
plt.xlabel('k')
plt.ylabel('Cross Entropy')
plt.show()
def run_logistic_regression_small(hyperparameters):
# TODO specify training data
train_inputs_1, train_targets_1 = load_train_small()
valid_inputs, valid_targets = load_valid()
#PASS test_inputs AND test_targets IN PLACE OF VALIDATION DATASET TO RUN LOGISTIC REGRESSION ON TEST DATASET.
test_inputs, test_targets = load_test()
# N is number of examples; M is the number of features per example.
N, M = train_inputs_1.shape
# Logistic regression weights
# TODO:Initialize to random weights here.
#Seed same value
#np.random.seed(1)
weights = np.random.randn(M + 1, 1)
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
logging = np.zeros((hyperparameters['num_iterations'], 5))
iterations = np.zeros((hyperparameters['num_iterations']))
ce_training_1 = np.zeros((hyperparameters['num_iterations']))
ce_validation = np.zeros((hyperparameters['num_iterations']))
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs_1, train_targets_1,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets_1, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
#
#For TEST DATASET, pass test_inputs instead of valid_inputs and test_targets instead of valid_targets.
#
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
np.set_printoptions(threshold='nan')
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t + 1, f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100)
logging[t] = [
f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100
]
iterations[t] = t
ce_training_1[t] = cross_entropy_train
ce_validation[t] = cross_entropy_valid
plt.figure(1)
plt.plot(iterations, ce_training_1, marker='+', label="Training Set")
plt.plot(iterations, ce_validation, marker='o', label="Validation Set")
plt.legend(loc=2)
if (hyperparameters['weight_regularization'] == False):
plt.title('Logistic Regression - mnist_train_small')
else:
plt.title('Regularized Logistic Regression - mnist_train_small')
plt.xlabel('Iterations')
plt.ylabel('Cross Entropy')
plt.show()
return logging
def run_logistic_regression(hyperparameters):
# specify training data
xIn = False
while xIn == False:
x = raw_input('Training Set LARGE or SMALL? ')
print(x)
if x == 'LARGE':
print("HELLO")
train_inputs, train_targets = load_train()
xIn = True
elif x == 'SMALL':
print("hello")
train_inputs, train_targets = load_train_small()
xIn = True
else:
print("Please input LARGE or SMALL")
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
# N is number of examples; M is the number of features per example.
N, M = train_inputs.shape
print("N:", N, " M:", M)
# Logistic regression weights
# Initialize to random weights here.
weights = np.random.normal(0, 0.001, (M+1, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
logging = np.zeros((hyperparameters['num_iterations'], 5))
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
# print some stats
print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t+1, f / N, cross_entropy_train, frac_correct_train*100,
cross_entropy_valid, frac_correct_valid*100)
logging[t] = [f / N, cross_entropy_train, frac_correct_train*100, cross_entropy_valid, frac_correct_valid*100]
return logging
def run_logistic_regression(hyperparameters, mode):
train_inputs = []
train_targets = []
valid_inputs, valid_targets = load_valid()
if mode == "normal":
train_inputs, train_targets = load_train()
elif mode == "small":
train_inputs, train_targets = load_train_small()
N, M = train_inputs.shape
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.zeros((M + 1, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
iteration_list = []
cross_entropy_train_list = []
cross_entropy_valid_list = []
frac_correct_train_list = []
frac_correct_valid_list = []
# Begin learning with gradient descent
for t in range(hyperparameters['num_iterations']):
iteration_list.append(t)
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
print(
("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t + 1, f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100))
cross_entropy_train_list.append(cross_entropy_train)
cross_entropy_valid_list.append(cross_entropy_valid)
frac_correct_train_list.append(frac_correct_train)
frac_correct_valid_list.append(frac_correct_valid)
plt.title("Logistic Regression with mnist_train: Cross Entropy")
plt.plot(iteration_list,
cross_entropy_train_list,
label="cross entropy train")
plt.plot(iteration_list,
cross_entropy_valid_list,
label='cross entropy validation')
plt.legend(loc="upper right")
plt.show()
plt.title(
"Logistic Regression with mnist_train: Correct Classification Rate")
plt.plot(iteration_list,
frac_correct_train_list,
label="frac correct train")
plt.plot(iteration_list,
frac_correct_valid_list,
label='frac correct validation')
plt.legend(loc="upper right")
plt.show()
cell_text = [[cross_entropy_train_list[-1], cross_entropy_valid_list[-1]],
[frac_correct_train_list[-1], frac_correct_valid_list[-1]]]
rows = ['ce', 'fac rate']
cols = ['train set', 'validation set']
colors = plt.cm.BuPu(np.linspace(0, 0.5, len(rows)))
colors = colors[::-1]
fig, axs = plt.subplots(2, 1)
axs[0].axis('tight')
axs[0].axis('off')
axs[0].table(cellText=cell_text,
colLabels=cols,
rowLabels=rows,
loc='top',
rowColours=colors)
plt.subplots_adjust(left=0.3, top=0.8)
plt.show()
def run_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
test_inputs, test_targets = load_test()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': .05,
'weight_regularization': .5,
'num_iterations': 100
}
# Logistic regression weights
#weights = np.random.rand(len(train_inputs[0]) + 1, 1)
#weights = (np.random.rand(len(train_inputs[0]) + 1, 1) - 0.5) * 2
#weights = np.zeros((len(train_inputs[0])+1, 1))
weights = np.ones((len(train_inputs[0])+1, 1)) - .9
#weights = np.reshape(np.append(np.mean(train_inputs, axis=0), np.mean(np.mean(train_inputs, axis=0))), (len(train_inputs[0])+1, 1))
#weights = np.ones((len(train_inputs[0])+1, 1)) - np.zeros((len(train_inputs[0])+1, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
training_ce = []
valid_ce = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
training_ce.append(cross_entropy_train)
valid_ce.append(cross_entropy_valid)
predictions_test = logistic_predict(weights, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets, predictions_test)
print(str(cross_entropy_test) + ', ' + str((frac_correct_test * 100)))
plt.plot(range(1, hyperparameters["num_iterations"]+1), training_ce, label="cross entropy train")
plt.plot(range(1, hyperparameters["num_iterations"]+1), valid_ce, label="cross entropy valid")
plt.legend()
plt.show()
def run_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_test()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.01,
'weight_regularization': 10,
'num_iterations': 400
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = []
for _ in range(train_inputs.shape[1] + 1):
weights.append(random.uniform(0.01, 0.2))
weights = np.array(weights)
x = weights.shape[0]
weights = weights.reshape((x, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
ceChartTrain = []
ceChartValid = []
errChartTrain = []
errChartValid = []
iteration = []
# Begin learning with gradient descent
for t in range(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
iteration.append(t)
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
errChartTrain.append(1 - frac_correct_train)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
errChartValid.append(1 - frac_correct_valid)
# print some stats
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} \n TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
.format(t + 1, f / N, cross_entropy_train,
frac_correct_train * 100, cross_entropy_valid,
frac_correct_valid * 100))
plt.plot(iteration, errChartTrain, label="Train Acc")
plt.plot(iteration, errChartValid, label="Valid Acc")
plt.xlabel("Iteration")
plt.ylabel("Accuracy")
plt.legend()
plt.show()
def pen_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
weight_reg_vec = [ pow(10, i) for i in range(-5, 0, 1)]
N, M = train_inputs.shape
class_error_train = []
class_error_val = []
cross_en_train = []
cross_en_val = []
for lam in weight_reg_vec:
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.85,
'weight_regularization': lam,
'num_iterations': 30
}
# Verify that your logistic :function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
ce_train_vec = []
ce_val_vec = []
corr_train_vec = []
corr_val_vec = []
for i in range(0, 5):
# Logistic regression weights
weights = np.random.rand(M+1, 1)
#weights = np.zeros(M+1)
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic_pen(weights, train_inputs, train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * (df + lam * weights)/ N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
# add data for plots
# add final CEs and errors
ce_train_vec.append(float(cross_entropy_train))
ce_val_vec.append(float(cross_entropy_valid))
corr_train_vec.append(frac_correct_train)
corr_val_vec.append(frac_correct_valid)
# average across multiple runs (due to random initials)
cross_en_train.append(float(np.average(ce_train_vec)))
cross_en_val.append(float(np.average(ce_val_vec)))
class_error_train.append(1.0-float(np.average(corr_train_vec)))
class_error_val.append(1.0-float(np.average(corr_val_vec)))
x = np.array(weight_reg_vec)
plt.semilogx(x, cross_en_train, 'r', label='Training')
plt.semilogx(x, cross_en_val, 'b', label='Validation')
plt.xlabel('Regularization factor')
plt.ylabel('Cross Entropy')
plt.legend(['Training', 'Validation'], bbox_to_anchor=(1, 0.5))
plt.show()
plt.semilogx(x, class_error_train, 'r', label='Training')
plt.semilogx(x, class_error_val, 'b:', label='Validation')
plt.xlabel('Regularization factor')
plt.ylabel('Classification Error')
plt.legend(['Training', 'Validation'], bbox_to_anchor=(1, 0.5))
plt.show()
test_inputs, test_targets = load_test()
hyperparameters['weight_regularization'] = 0.01
hyperparameters['num_iterations'] = 30
f, df, predictions = logistic(weights, test_inputs, test_targets, hyperparameters)
cross_entropy_train, frac_correct_train = evaluate(test_targets, predictions)
stat_msg = "ITERATION:{:4d} TEST NLOGL:{:4.2f} TEST CE:{:.6f} TEST FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100))
def run_logistic_regression():
#train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.99,
'weight_regularization': 0.1,
'num_iterations': 200
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.normal(0, math.sqrt(0.01),
train_inputs.shape[1] + 1).reshape(-1, 1)
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
valid_set = []
log_likelihood = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
log_likelihood.append(f)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
valid_set.append(frac_correct_valid)
# print some stats
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t + 1, f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100)
plt.plot(range(hyperparameters['num_iterations']), log_likelihood)
plt.plot(range(hyperparameters['num_iterations']), valid_set)
plt.axis([0, 140, -10, 140])
#splt.axis([0, 30, -1, 30])
plt.show()
def run_logistic_regression(train_inputs, train_targets, training_type, lr,
iteration, lambd, isPenalized):
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': lr,
'weight_regularization': lambd,
'num_iterations': iteration,
}
# Logistic regression weights
weights = np.random.uniform(-0.05, 0.05, M + 1)
# weights = np.random.rand(M+1)
# w = [0.001]*(M+1)
# weights = np.asarray(w)
weights = weights.reshape((M + 1, 1))
# weights = np.zeros((M+1, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
# run_check_grad(hyperparameters)
train_ce = []
train_frac = []
valid_ce = []
valid_frac = []
test_ce = []
test_frac = []
# Begin learning with gradient descent
for t in range(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
if (isPenalized == False):
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
else:
f, df, predictions = logistic_pen(weights, train_inputs,
train_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
predictions_test = logistic_predict(weights, test_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
cross_entropy_test, frac_correct_test = evaluate(
test_targets, predictions_test)
# print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
# "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}".format(
# t+1, f / N, cross_entropy_train, frac_correct_train*100,
# cross_entropy_valid, frac_correct_valid*100))
# print("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TEST CE{:.6f} TEST FRAC:{:2.2f}".format(
# t+1, f/N, cross_entropy_test, frac_correct_test*100)
# )
train_ce.append(cross_entropy_train)
train_frac.append(frac_correct_train * 100)
valid_ce.append(cross_entropy_valid)
valid_frac.append(frac_correct_valid * 100)
test_ce.append(cross_entropy_test)
test_frac.append(frac_correct_test * 100)
# print(">>>>>>>>>")
# print("smallest TRAIN CE: {:.6f} at iteration {:d} smallest VALID CE: {:.6f} at iteration {:d}".format(min(train_ce), train_ce.index(min(train_ce))+1, min(valid_ce), valid_ce.index(min(valid_ce))+1))
# x = np.arange(1,hyperparameters['num_iterations']+1, 1)
# fig, ax = plt.subplots()
# ax.grid(False)
# ax.plot(x, train_ce, label = "training_ce")
# ax.plot(x, valid_ce, label = "valid_ce")
# ax.scatter(train_ce.index(min(train_ce))+1, min(train_ce), label="smallest train_ce")
# ax.scatter(valid_ce.index(min(valid_ce))+1, min(valid_ce), label="smallest valid_ce")
# ax.legend(loc='upper right')
# ax.set(xlabel = "Iteration", ylabel = "Cross Entropy")
# fig.suptitle("Cross Entropy Error Processing on Dataset "+ training_type +" LR "+str(hyperparameters["learning_rate"])+ " Penalty "+str(hyperparameters["weight_regularization"]), fontsize=9)
# # plt.show()
# fig.savefig(training_type+"Learning Rate "+str(hyperparameters["learning_rate"])+ " Penalty "+str(hyperparameters["weight_regularization"])+" Iteration "+str(hyperparameters["num_iterations"])+".png")
return train_ce, valid_ce, test_ce, train_frac, valid_frac, test_frac
def run_logistic_regression(r):
# Indicate which training set you used:
# Uncomment out for training set.
train_inputs, train_targets = load_train()
training_set = ''
# Uncomment out for the small training set.
#train_inputs, train_targets = load_train_small() # 'Small'
#training_set = 'Small'
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': r,
'weight_regularization': 1,
'num_iterations': 200
}
# Logistic regression weights
# TODO:Initialize weights here.
# fixed weights
# weights = np.array([0.01]*(M+1)).reshape(M+1,1)
# random weights:
weights = np.transpose([np.random.normal(0, 0.1, M + 1)])
#Initialize to random weights here.
#weights = np.transpose([np.random.normal(0, hyperparameters['weight_regularization'], M+1)])
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
ce_train = []
ce_valid = []
fc_train = []
fc_valid = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# Uncomment to evaluate the prediction for test set
prediction_test = logistic_predict(weights, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(
test_targets, prediction_test)
print "\n test ce: ", str(
cross_entropy_test), "test frac_corr ", frac_correct_test
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t + 1, float(f / N), float(cross_entropy_train),
float(frac_correct_train * 100),
float(cross_entropy_valid),
float(frac_correct_valid * 100))
ce_train = np.append(ce_train, cross_entropy_train)
ce_valid = np.append(ce_valid, cross_entropy_valid)
fc_train = np.append(fc_train, frac_correct_train)
fc_valid = np.append(fc_valid, frac_correct_valid)
mp.plot(ce_train, label="training", linestyle="--")
mp.plot(ce_valid, label="test", linewidth=4)
mp.xlabel("num_iteration")
mp.ylabel("cross entropy")
mp.title("Cross Entropy of Training Set " + training_set +
"\n with learning rate" + str(hyperparameters['learning_rate']))
mp.savefig("2.2_ce_" + str(hyperparameters['learning_rate']) +
training_set + ".png")
mp.clf()
def run_logistic_regression():
train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs, test_targets = load_test()
N, M = train_inputs.shape
hyperparameters = {
'learning_rate': 0.1,
'weight_regularization': 0,
'num_iterations': 200
}
# Logistic regression weights
weights = np.random.rand(M + 1, 1) / 10
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
TRAIN_CE = []
VALID_CE = []
# Begin learning with gradient descent
for t in range(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
f_test, df_test, predictions_test = logistic(weights, test_inputs,
test_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
cross_entropy_test, frac_correct_test = evaluate(
test_targets, predictions_test)
TRAIN_CE.append(cross_entropy_train[0][0])
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
VALID_CE.append(cross_entropy_valid[0][0])
# print some stats
print("ITERATION:{} TRAIN NLOGL:{} TRAIN CE:{} "
"TRAIN FRAC:{} VALID CE:{} VALID FRAC:{}".format(
t + 1, f / N, cross_entropy_train[0][0],
frac_correct_train * 100, cross_entropy_valid[0][0],
frac_correct_valid * 100))
print("ITERATION:{} TEST NLOGL:{} TEST CE:{} "
"TEST FRAC:{}".format(t + 1, f_test / test_targets.shape[0],
cross_entropy_test[0][0],
frac_correct_test * 100))
plt.scatter(np.arange(1, hyperparameters['num_iterations'] + 1), TRAIN_CE)
plt.scatter(np.arange(1, hyperparameters['num_iterations'] + 1), VALID_CE)
plt.legend(
['Cross entropy for train data', 'Cross entropy for validation data'])
plt.show()
def run_logistic_regression(hyperparameters, small=False, test=False):
# TODO specify training data
if small == False:
train_inputs, train_targets = load_train()
else:
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
# N is number of examples; M is the number of features per example.
N, M = train_inputs.shape
# Logistic regression weights
# TODO:Initialize to random weights here.
# weights = np.random.seed(0)
weights = np.random.normal(0, 0.05, (M + 1, 1))
# weights = np.zeros((M+1, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
logging = np.zeros((hyperparameters['num_iterations'], 5))
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
if t % 10 == 0:
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
).format(t + 1, f / N, cross_entropy_train,
frac_correct_train * 100, cross_entropy_valid,
frac_correct_valid * 100)
logging[t] = [
f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100
]
# Calculate cross entropy and classfication rate of the test set
if test == True and t == hyperparameters['num_iterations'] - 1:
test_inputs, test_targets = load_test()
predictions_test = logistic_predict(weights, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(
test_targets, predictions_test)
print("TEST CE:{:.6f} TEST FRAC:{:2.2f}").format(
cross_entropy_test, frac_correct_test * 100)
return logging, N
def run_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
test_inputs, test_targets = load_test()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': .05,
'weight_regularization': .5,
'num_iterations': 100
}
# Logistic regression weights
#weights = np.random.rand(len(train_inputs[0]) + 1, 1)
#weights = (np.random.rand(len(train_inputs[0]) + 1, 1) - 0.5) * 2
#weights = np.zeros((len(train_inputs[0])+1, 1))
weights = np.ones((len(train_inputs[0]) + 1, 1)) - .9
#weights = np.reshape(np.append(np.mean(train_inputs, axis=0), np.mean(np.mean(train_inputs, axis=0))), (len(train_inputs[0])+1, 1))
#weights = np.ones((len(train_inputs[0])+1, 1)) - np.zeros((len(train_inputs[0])+1, 1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
training_ce = []
valid_ce = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t + 1, float(f / N), float(cross_entropy_train),
float(frac_correct_train * 100),
float(cross_entropy_valid),
float(frac_correct_valid * 100))
training_ce.append(cross_entropy_train)
valid_ce.append(cross_entropy_valid)
predictions_test = logistic_predict(weights, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets,
predictions_test)
print(str(cross_entropy_test) + ', ' + str((frac_correct_test * 100)))
plt.plot(range(1, hyperparameters["num_iterations"] + 1),
training_ce,
label="cross entropy train")
plt.plot(range(1, hyperparameters["num_iterations"] + 1),
valid_ce,
label="cross entropy valid")
plt.legend()
plt.show()
def run_logistic_regression(hyperparameters):
# TODO specify training data
train_inputs, train_targets = load_train()
train_inputs_small, train_targets_small=load_train_small()
valid_inputs, valid_targets = load_valid()
test_inputs,test_targets=load_test()
# N is number of examples; M is the number of features per example.
N, M = train_inputs.shape
Ns,Ms= train_inputs_small.shape
# Logistic regression weights
# TODO:Initialize to random weights here.
#weights = 0.05*np.ones((M+1,1))
weights = 0.1*np.random.randn(M+1,1)
weightssmall=copy.deepcopy(weights)
weightspen=copy.deepcopy(weights)
weightspensmall=copy.deepcopy(weights)
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
logging = np.zeros((hyperparameters['num_iterations'], 9))
loggingsmall= np.zeros((hyperparameters['num_iterations'], 9))
loggingpen= np.zeros((hyperparameters['num_iterations'], 9))
loggingpensmall= np.zeros((hyperparameters['num_iterations'], 9))
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
validf, validdf, validpredictions = logistic(weights, valid_inputs, valid_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
predictions_test=logistic_predict(weights, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets, predictions_test)
# print some stats
# print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
# "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
# t+1, float(f / N), float(cross_entropy_train), float(frac_correct_train*100),
# float(cross_entropy_valid), float(frac_correct_valid*100))
logging[t] = [f / N, cross_entropy_train, frac_correct_train*100, cross_entropy_valid, frac_correct_valid*100,cross_entropy_test,frac_correct_test*100,f,validf]
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weightssmall, train_inputs_small, train_targets_small, hyperparameters)
validf, validdf, validpredictions = logistic(weightssmall, valid_inputs, valid_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets_small, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weightssmall = weightssmall - hyperparameters['learning_rate'] * df / Ns
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weightssmall, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
predictions_test=logistic_predict(weightssmall, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets, predictions_test)
# print some stats
#print "this is for smalltrainset"
#print ("ITERATION:{:4d} SMALLTRAIN NLOGL:{:4.2f} SMALLTRAIN CE:{:.6f} "
#"SMALLTRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
# t+1, float(f / N), float(cross_entropy_train), float(frac_correct_train*100),
# float(cross_entropy_valid), float(frac_correct_valid*100))
loggingsmall[t] = [f / Ns, cross_entropy_train, frac_correct_train*100, cross_entropy_valid, frac_correct_valid*100,cross_entropy_test,frac_correct_test*100,f,validf]
hyperparameters['weight_regularization']=True
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weightspen, train_inputs, train_targets, hyperparameters)
validf, validdf, validpredictions = logistic(weightspen, valid_inputs, valid_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weightspen = weightspen - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weightspen, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
predictions_test=logistic_predict(weightspen, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets, predictions_test)
# print some stats
# print ("ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
# "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
# t+1, float(f / N), float(cross_entropy_train), float(frac_correct_train*100),
# float(cross_entropy_valid), float(frac_correct_valid*100))
loggingpen[t] = [f / N, cross_entropy_train, frac_correct_train*100, cross_entropy_valid, frac_correct_valid*100,cross_entropy_test,frac_correct_test*100,f,validf]
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weightspensmall, train_inputs_small, train_targets_small, hyperparameters)
validf, validdf, validpredictions = logistic(weightspensmall, valid_inputs, valid_targets, hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets_small, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weightspensmall = weightspensmall - hyperparameters['learning_rate'] * df / Ns
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weightspensmall, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
predictions_test=logistic_predict(weightspensmall, test_inputs)
cross_entropy_test, frac_correct_test = evaluate(test_targets, predictions_test)
# print some stats
#print "this is for smalltrainset"
#print ("ITERATION:{:4d} SMALLTRAIN NLOGL:{:4.2f} SMALLTRAIN CE:{:.6f} "
#"SMALLTRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
# t+1, float(f / N), float(cross_entropy_train), float(frac_correct_train*100),
# float(cross_entropy_valid), float(frac_correct_valid*100))
loggingpensmall[t] = [f / N, cross_entropy_train, frac_correct_train*100, cross_entropy_valid, frac_correct_valid*100,cross_entropy_test,frac_correct_test*100,f,validf]
return logging,loggingsmall,loggingpen,loggingpensmall
def run_logistic_regression():
# train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
N, M = np.shape(train_inputs)
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.01,
'weight_regularization':...,
'num_iterations': 1000
}
# Logistic regression weights
# TODO:Initialize to random weights here.
# np.random.seed(0)
weights = np.random.rand(M + 1, 1) * 0.1
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# TODO: you may need to modify this loop to create plots, etc.
train_frac = []
valid_frac = []
ce_train = []
ce_valid = []
# Begin learning with gradient descent
for t in range(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print(
stat_msg.format(t + 1, float(f / N), float(cross_entropy_train),
float(frac_correct_train * 100),
float(cross_entropy_valid),
float(frac_correct_valid * 100)))
train_frac.append(frac_correct_train * 100)
valid_frac.append(frac_correct_valid * 100)
ce_train.append(cross_entropy_train)
ce_valid.append(cross_entropy_valid)
plt.plot(list(range(hyperparameters['num_iterations'])),
ce_train,
color='blue')
plt.plot(list(range(hyperparameters['num_iterations'])),
ce_valid,
color='green')
plt.title('logistic regression(learning rate = {})'.format(
hyperparameters['learning_rate']))
plt.xlabel('the number of iteration')
plt.ylabel('cross entropy')
plt.legend(['train(%1.2f)' % ce_train[-1], 'valid(%1.2f)' % ce_valid[-1]])
plt.show()
def run_logistic_regression():
#train_inputs, train_targets = load_train()
train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': 0.85,
'weight_regularization': 0.01,
'num_iterations': 30
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.rand(M+1, 1)
#weights = np.zeros(M+1)
# Verify that your logistic :function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
ce_train_vec = []
ce_val_vec = []
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
#f, df, predictions = logistic(weights, train_inputs, train_targets, hyperparameters)
f, df, predictions = logistic_pen(weights, train_inputs, train_targets, hyperparameters)
#print 'f, df, pred', f, df, predictions
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
#weights = weights - hyperparameters['learning_rate'] * df / N
weights = weights - hyperparameters['learning_rate'] * (df + hyperparameters['weight_regularization'] * weights)/ N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
# print some stats
stat_msg = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
# add data for plots
ce_train_vec.append(float(cross_entropy_train))
ce_val_vec.append(float(cross_entropy_valid))
x = range(1, len(ce_train_vec)+1)
plt.plot(x, ce_train_vec, 'r', x, ce_val_vec, 'b')
plt.xlabel('Iterations')
plt.ylabel('Cross Entropy')
plt.legend(['Training', 'Validation'], bbox_to_anchor=(1, 0.5))
plt.show()
test_inputs, test_targets = load_test()
f, df, predictions = logistic(weights, test_inputs, test_targets, hyperparameters)
cross_entropy_train, frac_correct_train = evaluate(test_targets, predictions)
stat_msg = "ITERATION:{:4d} TEST NLOGL:{:4.2f} TEST CE:{:.6f} TEST FRAC:{:2.2f}"
print stat_msg.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100))
Created on Sun Jun 1 15:22:25 2014
@author: sbroad
"""
from logistic import *
from newton import *
# investigate the basin of attraction for each of several logistic parameters
init = np.random.random(200)
plt.close(1)
plt.figure(1)
for a in [(0.5,'b.'), (1.5,'g.'), (2.5,'r.'), (3.2,'k.')]:
attr = [ logistic(x, a[0], n=100) for x in init]
plt.plot(init, attr,a[1],label=r'$\lambda=%.2f$' % (a[0]))
plt.ylim(ymin=0,ymax=1)
plt.legend(loc='best')
# investigate the basin of attraction for roots of sin(x)
init = np.random.random(200)*10
plt.close(2)
plt.figure(2)
quad = (lambda x: x**2-6*x+8, 'b.', r'$x^2-5x+4$')
trig = (lambda x: sin(x/2.), 'g.', r'$\sin(\pi x)$')
tran = (lambda x: log(x),'r.', r'$\log x$')
for b in [quad, trig, tran]:
attr = [newton(b[0], x, n=100) for x in init]
def run_logistic_regression(hyps, small=False, pen=False, to_print=False):
if small:
train_inputs, train_targets = load_train_small()
else:
train_inputs, train_targets = load_train()
valid_inputs, valid_targets = load_valid()
N, M = train_inputs.shape
# TODO: Set hyperparameters
hyperparameters = {
'learning_rate': hyps[_LR],
'weight_regularization': hyps[_WR],
'num_iterations': hyps[_NOI]
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.random(M+1) - 0.5
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
# run_check_grad(hyperparameters)
# Begin learning with gradient descent
max_correct_train = (0, 0)
max_correct_valid = (0, 0)
min_cE_valid = (1000000, 0)
CE_vec_valid = np.zeros(shape=(hyperparameters['num_iterations'], ))
CE_vec_train = np.zeros(shape=(hyperparameters['num_iterations'], ))
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
if pen:
f, df, predictions = logistic_pen(weights, train_inputs, train_targets, hyperparameters)
else:
f, df, predictions = logistic(weights, train_inputs, train_targets, [])
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
df.shape = (M+1, )
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(valid_targets, predictions_valid)
CE_vec_valid[t] = cross_entropy_valid
CE_vec_train[t] = cross_entropy_train
if frac_correct_train > max_correct_train[0]:
max_correct_train = frac_correct_train, t
if frac_correct_valid > max_correct_valid[0]:
max_correct_valid = frac_correct_valid, t
if cross_entropy_valid < min_cE_valid[0]:
min_cE_valid = cross_entropy_valid, t
# print some stats
if to_print:
stat_msg1 = "ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
stat_msg1 += "TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}"
print stat_msg1.format(t+1,
float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100))
if t == hyperparameters['num_iterations'] - 1:
return (float(f / N),
float(cross_entropy_train),
float(frac_correct_train*100),
float(cross_entropy_valid),
float(frac_correct_valid*100),
max_correct_train,
max_correct_valid,
min_cE_valid,
CE_vec_train, CE_vec_valid,
)
def run_logistic_regression(hyperparameters):
global iterations, valid_or_test
#Comment out one of these based on small or large training set
train_inputs, train_targets = load_train()
# train_inputs, train_targets = load_train_small()
#Comment out one set of these based on validation or test set
valid_inputs, valid_targets = load_valid()
valid_or_test = 0
# valid_inputs, valid_targets = load_test()
# valid_or_test = 1
# N is number of examples; M is the number of features per example.
N, M = train_inputs.shape
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.zeros((M + 1, 1))
#weights = np.random.randn(M+1, 1)
#weights = np.random.randint(0,10, (M+1,1))
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
# Begin learning with gradient descent
logging = np.zeros((hyperparameters['num_iterations'], 5))
for t in xrange(hyperparameters['num_iterations']):
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
# weights_old = weights
weights = weights - hyperparameters['learning_rate'] * df / N
# if sum(abs(weights_old - weights)) > 0.05:
# pass
# else:
# iterations = t
# break
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
#print some stats
#print t+1, f / N, cross_entropy_train, frac_correct_train, cross_entropy_valid, frac_correct_valid
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t + 1, float(f / N), float(cross_entropy_train),
float(frac_correct_train) * 100, float(cross_entropy_valid),
float(frac_correct_valid) * 100)
logging[t] = [
f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100
]
return logging
def back_prop_learning(ex_data, ex_label):
n = random.randint(5, 10)
w = [[0 for j in range(n + 3)] for i in range(n + 2)]
b = [0 for j in range(n + 3)]
layer = [[0, 1], [i for i in range(2, n + 2)], [n + 2]]
alpha = 0.1
#2 nodes in input layer, those are coordinates of points
#n nodes in hidden layer
for i in range(n):
for j in range(n + 3):
w[i][j] = random.uniform(-1, 1)
for j in range(n + 3):
b[j] = random.uniform(-1, 1)
for k in range(len(ex_data)):
x = ex_data[k]
y = ex_label[k]
a = [0 for i in range(2 + n + 1)]
delta = [0 for i in range(2 + n + 1)]
vector_in = [0 for i in range(2 + n + 1)]
# print("x = ", x )
for i in range(2):
a[i] = x[i]
# for each node j in hidden layer
# n nodes in the hidden layer are numbered from 0 -> n + 1#for each node j in hidden layer
# # n nodes in the hidden layer are numbered from 0 -> n + 1
for j in range(2, n + 2):
vector_in[j] = 0
for i in range(2):
vector_in[j] = vector_in[j] + w[i][j] * a[i]
a[j] = logistic(vector_in[j] + b[j])[1]
#calculate output node
j = n + 2
vector_in[j] = 0.0
for i in range(2, n + 2):
vector_in[j] = vector_in[j] + w[i][j] * a[i]
a[n + 2] = logistic(vector_in[j] + b[n + 2])[1]
delta[j] = y - a[n + 2]
for l in range(1, -1, -1):
for i in layer[l]:
g = logistic(vector_in[i] + b[i])[1]
if l == 1:
delta[i] = g * (
1 - g) * w[i][j] * delta[n + 2] #one output node j
if l == 0:
delta[i] = g * (1 - g) * sum(
[w[i][j] * delta[j]
for j in range(2, n + 2)]) #one output node j
for j in range(2, n + 2): #foreach j in 2-> n+1
#from first input layer to hidden layer
w[0][j] = w[0][j] + alpha * a[0] * delta[j]
w[1][j] = w[1][j] + alpha * a[1] * delta[j]
#from hidden layer to output layer
w[j][n + 2] = w[j][n + 2] + alpha * a[j] * delta[n + 2]
for j in range(2, n + 3):
b[j] = b[j] + alpha * delta[j]
return w, b, n
def run_logistic_regression():
train_inputs, train_targets = load_train()
#train_inputs, train_targets = load_train_small()
valid_inputs, valid_targets = load_valid()
#valid_inputs, valid_targets = load_test()
N, M = train_inputs.shape
hyperparameters = {
'learning_rate': 0.01,
'weight_regularization': 0.1,
'num_iterations': 1000
}
# Logistic regression weights
# TODO:Initialize to random weights here.
weights = np.random.randn(M + 1, 1) / 10
# Verify that your logistic function produces the right gradient.
# diff should be very close to 0.
run_check_grad(hyperparameters)
ce_train = []
ce_valid = []
# Begin learning with gradient descent
for t in xrange(hyperparameters['num_iterations']):
# TODO: you may need to modify this loop to create plots, etc.
# Find the negative log likelihood and its derivatives w.r.t. the weights.
f, df, predictions = logistic(weights, train_inputs, train_targets,
hyperparameters)
# Evaluate the prediction.
cross_entropy_train, frac_correct_train = evaluate(
train_targets, predictions)
if np.isnan(f) or np.isinf(f):
raise ValueError("nan/inf error")
# update parameters
weights = weights - hyperparameters['learning_rate'] * df / N
# Make a prediction on the valid_inputs.
predictions_valid = logistic_predict(weights, valid_inputs)
# Evaluate the prediction.
cross_entropy_valid, frac_correct_valid = evaluate(
valid_targets, predictions_valid)
# print some stats
print(
"ITERATION:{:4d} TRAIN NLOGL:{:4.2f} TRAIN CE:{:.6f} "
"TRAIN FRAC:{:2.2f} VALID CE:{:.6f} VALID FRAC:{:2.2f}").format(
t + 1, f / N, cross_entropy_train, frac_correct_train * 100,
cross_entropy_valid, frac_correct_valid * 100)
ce_train.append(cross_entropy_train)
ce_valid.append(cross_entropy_valid)
pyplot.title("Logistic Regression with mnist_train")
pyplot.xlabel("Training Iteration")
pyplot.ylabel("Cross Entropy")
pyplot.plot(xrange(1, hyperparameters['num_iterations'] + 1),
ce_train,
label="training cross entropy")
pyplot.plot(xrange(1, hyperparameters['num_iterations'] + 1),
ce_valid,
label="validation cross entropy")
pyplot.legend()
pyplot.show()
