def main(optimizer='gd'):
# load and plot the data points
train_X, train_Y = load_dataset()
plt.show()
layers_dims = [train_X.shape[0], 5, 2, 1]
# try different optimization methods
if optimizer == 'gd':
# mini-batch gradient descent
parameters = model(train_X, train_Y, layers_dims, optimizer="gd")
plt.title("Model with Gradient Descent optimization")
elif optimizer == 'momentum':
# mini-batch gradient descent with momentum
parameters = model(train_X, train_Y, layers_dims, optimizer="momentum")
plt.title("Model with Momentum optimization")
elif optimizer == 'adam':
# mini-batch gradient descent with Adam
parameters = model(train_X, train_Y, layers_dims, optimizer="adam")
plt.title("Model with Adam optimization")
else:
print("No such optimization method named " + optimizer)
return
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
def main():
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
train_X, train_Y = load_dataset()
layers_dims = [train_X.shape[0], 5, 2, 1]
#parameters = model(train_X, train_Y, layers_dims, optimizer = "gd")
#parameters = model(train_X, train_Y, layers_dims, beta = 0.9, optimizer = "momentum")
parameters = model(train_X, train_Y, layers_dims, optimizer="adam")
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
#plt.title("Model with Gradient Descent optimization")
#plt.title("Model with Momentum optimization")
plt.title("Model with Adam optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
plt.show()
def train_adam():
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer="adam")
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
plt.title("Model with Adam optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
def mini_batch_with_gradient():
train_X, train_Y = load_dataset()
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer="gd")
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
def main():
train_X, train_Y = load_dataset()
layers_dims = [train_X.shape[0], 5, 2, 1]
print("training with mini-batch gd optimizer")
learned_parameters_gd = model(train_X,
train_Y,
layers_dims,
optimizer="gd")
predict(train_X, train_Y, learned_parameters_gd)
# Plot decision boundary
plt.title("model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(learned_parameters_gd, x.T),
train_X, train_Y)
print("training with momentum optimizer")
learned_parameters_momentum = model(train_X,
train_Y,
layers_dims,
beta=0.9,
optimizer="momentum")
predict(train_X, train_Y, learned_parameters_momentum)
# Plot decision boundary
plt.title("model with Momentum optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(
lambda x: predict_dec(learned_parameters_momentum, x.T), train_X,
train_Y)
print("training with adam optimizer")
learned_parameters_adam = model(train_X,
train_Y,
layers_dims,
optimizer="adam")
predict(train_X, train_Y, learned_parameters_adam)
# Plot decision boundary
plt.title("model with Adam optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(learned_parameters_adam, x.T),
train_X, train_Y)
return None
# plot the cost
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('epochs (per 100)')
plt.title("Learning rate = " + str(learning_rate))
plt.show()
return parameters
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer="gd")
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X,
train_Y,
layers_dims,
num_epochs=30000,
# You will now run this 3 layer neural network with each of the 3 optimization methods.
#
# ### 5.1 - Mini-batch Gradient descent
#
# Run the following code to see how the model does with mini-batch gradient descent.
# In[16]:
# train 3-layer model
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer = "gd")
# Predict
predictions = predict(train_X, train_Y, parameters)
# Plot decision boundary
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5,2.5])
axes.set_ylim([-1,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
# ### 5.2 - Mini-batch gradient descent with momentum
#
# Run the following code to see how the model does with momentum. Because this example is relatively simple, the gains from using momemtum are small; but for more complex problems you might see bigger gains.
# In[17]:
plt.xlabel("#epochs")
plt.title("Learning rate = " + str(learning_rate))
plt.show()
return params
# Test model using **common gradient descent**
# In[27]:
layer_dims = [train_X.shape[0], 5, 2, 1]
params = model(train_X, train_Y, layer_dims, optimizer="gd", is_plot=True)
# In[28]:
predictions = opt_utils.predict(train_X, train_Y, params)
plt.title("Gradient Descent")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
opt_utils.plot_decision_boundary(lambda x: opt_utils.predict_dec(params, x.T),
train_X, np.squeeze(train_Y))
# Test model using **momentum gradient descent**
# In[29]:
layer_dims = [train_X.shape[0], 5, 2, 1]
params = model(train_X,
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer="momentum", is_plot=True)
preditions = opt_utils.predict(train_X, train_Y, parameters)
# 绘制分类图
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
opt_utils.plot_decision_boundary(lambda x: opt_utils.predict_dec(parameters, x.T), train_X, train_Y)
'''
#测试adam优化器
train_X, train_Y = opt_utils.load_dataset(is_plot=False)
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X,
train_Y,
layers_dims,
optimizer="adam",
is_plot=True)
preditions = opt_utils.predict(train_X, train_Y, parameters)
# 绘制分类图
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5, 2.5])
axes.set_ylim([-1, 1.5])
opt_utils.plot_decision_boundary(
lambda x: opt_utils.predict_dec(parameters, x.T), train_X, train_Y)
print("Cost after iteration {}: {}".format(i, cost))
if print_cost and i % 1000 == 0:
costs.append(cost)
# plot the cost
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('iterations (x1,000)')
plt.title("Learning rate =" + str(learning_rate))
plt.show()
return parameters
parameters = model(train_X, train_Y)
print ("On the training set:")
predictions_train = predict(train_X, train_Y, parameters)
print ("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
plt.title("Model without regularization")
axes = plt.gca()
axes.set_xlim([-0.75,0.40])
axes.set_ylim([-0.75,0.65])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
#L2 Regularization
def compute_cost_with_regularization(A3, Y, parameters, lambd):
"""
Implement the cost function with L2 regularization. See formula (2) above.
costs.append(cost)
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('epoch (per 100)')
plt.title('Learning rate = {}'.format(learning_rate))
plt.show()
return parameters
#---------------------------------------------------
# 5.1. Mini-batch Gradient Descent
#---------------------------------------------------
layer_dims = [train_x.shape[0], 5, 2, 1]
parameters = model(train_x, train_y, layer_dims, optimizer='gd')
predictions = predict(train_x, train_y, parameters)
plt.title("Model with Gradient Descent optimization")
axes = plt.gca()
axes.set_xlim([-1.5,2.5])
axes.set_ylim([-1,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_x, train_y)
#---------------------------------------------------
# 5.2. Mini-batch Gradient Descent with momentum
#---------------------------------------------------
layer_dims = [train_x.shape[0], 5, 2, 1]
parameters = model(train_x, train_y, layer_dims, beta=0.9, optimizer='momentum')
predictions = predict(train_x, train_y, parameters)
