def main(initialization='he'):
train_X, train_Y, test_X, test_Y = load_dataset()
# try different initialization methods
if initialization == 'zero':
parameters = model(train_X, train_Y, initialization='zero')
elif initialization == 'random':
parameters = model(train_X, train_Y, initialization='random')
else:
parameters = model(train_X, train_Y, initialization='he')
# make predictions
print("-----The result of using " + initialization + "-----")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
# plot the result
plt.title("Model with " + initialization + " initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
def main():
# 读取并绘制数据: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()
# 选择一个初始化方法
parameters = initialize_parameters_he([2, 4, 1])
# 打印随机初始化后各参数的值
print("W1 = " + str(parameters["W1"]))
print("b1 = " + str(parameters["b1"]))
print("W2 = " + str(parameters["W2"]))
print("b2 = " + str(parameters["b2"]))
# 通过神经网络模型进行训练，得到参数
parameters = model(train_X, train_Y, initialization = "he")
# 用训练得到的参数进行预测，并分别输出训练集和测试集的预测准确率
print ("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print ("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
# 打印预测结果
print (predictions_train)
print (predictions_test)
# 绘制红蓝点分离情况图
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, np.squeeze(train_Y)) # 注意最后一个参数不能直接用train_Y，要加上np.squeeze()
def plt_decision_boundary(title):
plt.title(title)
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
plt.show()
def test_models_he():
parameters = model(train_X, train_Y, initialization = "he")
print ("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print ("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
def test_random():
parameters = initialize_parameters_random([3, 2, 1])
print("W1 = " + str(parameters["W1"]))
print("b1 = " + str(parameters["b1"]))
print("W2 = " + str(parameters["W2"]))
print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="random")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
print(predictions_train)
print(predictions_test)
plt.title("Model with large random initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
def main():
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
# load image dataset: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()
parameters = model(train_X, train_Y, initialization = "he")
print ("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print ("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
plt.figure()
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
plt.show()
def main():
train_X, train_Y, test_X, test_Y = load_dataset()
# not so good initialization
parameters = model(train_X, train_Y, initialization="random")
print("On the train set:")
predict(train_X, train_Y, parameters)
print("On the test set:")
predict(test_X, test_Y, parameters)
# good initialization
parameters = model(train_X, train_Y, initialization="he")
print("On the train set:")
predict(train_X, train_Y, parameters)
print("On the test set:")
predict(test_X, test_Y, parameters)
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
predictions_test = predict(test_X, test_Y, parameters)
# The performance is really bad, and the cost does not really decrease, and the algorithm performs no better than random guessing. Why? Lets look at the details of the predictions and the decision boundary:
# In[33]:
print("predictions_train = " + str(predictions_train))
print("predictions_test = " + str(predictions_test))
# In[34]:
plt.title("Model with Zeros initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
# The model is predicting 0 for every example.
#
# In general, initializing all the weights to zero results in the network failing to break symmetry. This means that every neuron in each layer will learn the same thing, and you might as well be training a neural network with $n^{[l]}=1$ for every layer, and the network is no more powerful than a linear classifier such as logistic regression.
# <font color='blue'>
# **What you should remember**:
# - The weights $W^{[l]}$ should be initialized randomly to break symmetry.
# - It is however okay to initialize the biases $b^{[l]}$ to zeros. Symmetry is still broken so long as $W^{[l]}$ is initialized randomly.
#
# ## 3 - Random initialization
#
# To break symmetry, lets intialize the weights randomly. Following random initialization, each neuron can then proceed to learn a different function of its inputs. In this exercise, you will see what happens if the weights are intialized randomly, but to very large values.
#
print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="zeros")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
print("predictions_train = " + str(predictions_train))
print("predictions_test = " + str(predictions_test))
plt.title("Model with Zeros initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
np.squeeze(train_Y))
def initialize_parameters_random(layers_dims):
"""
Arguments:
layer_dims -- python array (list) containing the size of each layer.
Returns:
parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL":
W1 -- weight matrix of shape (layers_dims[1], layers_dims[0])
b1 -- bias vector of shape (layers_dims[1], 1)
...
WL -- weight matrix of shape (layers_dims[L], layers_dims[L-1])
bL -- bias vector of shape (layers_dims[L], 1)
"""
# The performance is really bad, and the cost does not really decrease, and the algorithm performs no better than random guessing. Why? Lets look at the details of the predictions and the decision boundary:
# In[6]:
print ("predictions_train = " + str(predictions_train))
print ("predictions_test = " + str(predictions_test))
# In[7]:
plt.title("Model with Zeros initialization")
axes = plt.gca()
axes.set_xlim([-1.5,1.5])
axes.set_ylim([-1.5,1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, train_Y)
# The model is predicting 0 for every example.
#
# In general, initializing all the weights to zero results in the network failing to break symmetry. This means that every neuron in each layer will learn the same thing, and you might as well be training a neural network with $n^{[l]}=1$ for every layer, and the network is no more powerful than a linear classifier such as logistic regression.
# <font color='blue'>
# **What you should remember**:
# - The weights $W^{[l]}$ should be initialized randomly to break symmetry.
# - It is however okay to initialize the biases $b^{[l]}$ to zeros. Symmetry is still broken so long as $W^{[l]}$ is initialized randomly.
#
# ## 3 - Random initialization
#
# To break symmetry, lets intialize the weights randomly. Following random initialization, each neuron can then proceed to learn a different function of its inputs. In this exercise, you will see what happens if the weights are intialized randomly, but to very large values.
from nns.loss import CrossEntropy
from nns.layer import Layer
from nns.model import Model
if __name__ == '__main__':
model = Model(loss=CrossEntropy(), learning_rate=0.1)
model.append(Layer(train_X.shape[0], Relu()))
model.append(Layer(10, Sigmoid()))
model.append(Layer(5, Sigmoid()))
model.append(Layer(1, Sigmoid()))
model.compile()
model.train(train_X, train_Y, number_iterations=15000)
# parameters = model(train_X, train_Y, initialization="zeros", num_iterations=15000)
# import pprint
# pprint.pprint(parameters)
# print ("On the train set:")
# predictions_train = predict(train_X, train_Y, parameters)
# print ("On the test set:")
# predictions_test = predict(test_X, test_Y, parameters)
plt.title("Model with large random initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: model.predict(x.T) > 0.5, train_X,
train_Y)
plt.show()
# load image dataset: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()
plt.show()
n = DeepNeuralNetwork
n.Parameters_Init(n, [2, 10, 5, 1])
print("W1 = " + str(n.parameters["W1"]))
print("b1 = " + str(n.parameters["b1"]))
print("W2 = " + str(n.parameters["W2"]))
print("b2 = " + str(n.parameters["b2"]))
n.Train(n,
train_X,
train_Y,
learning_rate=0.01,
iterations=15000,
print_cost=True)
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: n.Predict(n, x.T), train_X,
np.squeeze(train_Y))
plt.show()
n.Query(n, test_X, test_Y)
# print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="he", is_plot=True)
print("训练集")
predictions_train = init_utils.predict(train_X, train_Y, parameters)
print("测试集")
predicitons_test = init_utils.predict(test_X, test_Y, parameters)
print("predictions_train = " + str(predictions_train))
print("predictions_test = " + str(predicitons_test))
plt.title("model with Zeros initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
init_utils.plot_decision_boundary(
lambda x: init_utils.predict_dec(parameters, x.T), train_X, train_Y)
# 正则化对算法影响
train_X, train_Y, test_X, test_Y = reg_utils.load_2D_dataset(is_plot=False)
# plt.show()
def compute_cost_with_regularization(A3, Y, parameters, lambd):
m = Y.shape[1]
W1 = parameters["W1"]
W2 = parameters["W2"]
W3 = parameters["W3"]
logprobs = np.multiply(-np.log(A3), Y) + np.multiply(
-np.log(1 - A3), 1 - Y)
cost = 1. / m * np.nansum(logprobs) + 1. / m * lambd * (
# parameters = model(train_X, train_Y, initialization = "random")
# print ("On the train set:")
# predictions_train = predict(train_X, train_Y, parameters)
# print ("On the test set:")
# predictions_test = predict(test_X, test_Y, parameters)
# print (predictions_train)
# print (predictions_test)
# plt.title("Model with large random initialization")
# axes = plt.gca()
# axes.set_xlim([-1.5,1.5])
# axes.set_ylim([-1.5,1.5])
# plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X, np.reshape(train_Y,-1))
sep("He")
parameters = initialize_parameters_he([2, 4, 1])
print("W1 = " + str(parameters["W1"]))
print("b1 = " + str(parameters["b1"]))
print("W2 = " + str(parameters["W2"]))
print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="he")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
np.reshape(train_Y, -1))
def main1():
#%matplotlib inline
plt.rcParams['figure.figsize'] = (7.0, 4.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
# load image dataset: blue/red dots in circles
train_X, train_Y, test_X, test_Y = load_dataset()
parameters = initialize_parameters_zeros([3, 2, 1])
print("W1 = " + str(parameters["W1"]))
print("b1 = " + str(parameters["b1"]))
print("W2 = " + str(parameters["W2"]))
print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="zeros")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
print("predictions_train = " + str(predictions_train))
print("predictions_test = " + str(predictions_test))
plt.title("Model with Zeros initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
parameters = initialize_parameters_random([3, 2, 1])
print("W1 = " + str(parameters["W1"]))
print("b1 = " + str(parameters["b1"]))
print("W2 = " + str(parameters["W2"]))
print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="random")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
print(predictions_train)
print(predictions_test)
plt.title("Model with large random initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
parameters = initialize_parameters_he([2, 4, 1])
print("W1 = " + str(parameters["W1"]))
print("b1 = " + str(parameters["b1"]))
print("W2 = " + str(parameters["W2"]))
print("b2 = " + str(parameters["b2"]))
parameters = model(train_X, train_Y, initialization="he")
print("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y)
predictions_test = predict(test_X, test_Y, parameters)
# The performance is really bad, and the cost does not really decrease, and the algorithm performs no better than random guessing.
#
# Look at detail of the predictions and the decision boundary
print("predictions_train = " + str(predictions_train))
print("predictions_test = " + str(predictions_test))
#
plt.title("Model with Zeros initialization") # plot the title
axes = plt.gca() # get current axis on current figure
axes.set_xlim([-1.5, 1.5]) # sets x-axis limits
axes.set_ylim([-1.5, 1.5]) # set y-axis limits
plot_decision_boundary(lambda x: predict_dec(parameters, x.T), train_X,
train_Y) # generated contour plot below
# The model is predicting 0 for every example.
#
# In general, initializing all the weights to zero results in the network failing to break symmetry. This means that every neuron in each layer will learn the same thing.
# - The weights $W^{[l]}$ should be initialized randomly to break symmetry.
#
### 3) Random initialization
#
# To break symmetry, lets intialize the weights randomly. Following random initialization, each neuron can then proceed to learn a different function of its inputs.
#
### initialize_parameters_random ###
L = len(layers_dims)
for l in range(1,L):
parameters['W'+str(l)] = np.random.randn(layers_dims[l],layers_dims[l-1])*np.sqrt(2/layers_dims[l-1])
parameters['b'+str(l)] = np.zeros((layers_dims[l],1))
#使用断言保证数据格式正确
assert(parameters['W'+str(l)].shape == (layers_dims[l],layers_dims[l-1]))
assert(parameters['b'+str(l)].shape == (layers_dims[l],1))
return parameters
parameters = model(train_X, train_Y, initialization = "he",is_polt=True)
print("训练集:")
predictions_train = init_utils.predict(train_X, train_Y, parameters)
print("测试集:")
init_utils.predictions_test = init_utils.predict(test_X, test_Y, parameters)
plt.title("Model with He initialization")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
init_utils.plot_decision_boundary(lambda x: init_utils.predict_dec(parameters, x.T), train_X, train_Y)
# 不同的初始化方法可能导致性能最终不同
#
# 随机初始化有助于打破对称，使得不同隐藏层的单元可以学习到不同的参数。
#
# 初始化时，初始值不宜过大。
#
# He初始化搭配ReLU激活函数常常可以得到不错的效果
predictions_test = init_utils.predict(test_X, test_Y, params)
# 观察预测值：
# In[8]:
print("predictions_train: " + str(predictions_train))
print("prediction_test: " + str(predictions_test))
# Plot Decision Boundary
plt.title("Zero init")
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
init_utils.plot_decision_boundary(
lambda x: init_utils.predict_dec(params, x.T), train_X,
np.squeeze(train_Y))
# 可以看到全部的预测值为0，没有达到分类的要求。
# #### 随机初始化
# In[9]:
def init_params_random(layer_dims):
params = {}
np.random.seed(3)
layer_num = len(layer_dims)
for i in range(1, layer_num):
params["W" +
# %%
# use 'he' to initial parameters
np.random.seed(1)
params = model(X_train, Y_train, alpha=0.06, loops=15000, init_method='he')
print('train set')
prediction = init_utils.predict(X_train, Y_train, params)
print('test set')
prediction = init_utils.predict(X_test, Y_test, params)
# %%
axes = plt.gca()
axes.set_xlim([-1.5, 1.5])
axes.set_ylim([-1.5, 1.5])
init_utils.plot_decision_boundary(
lambda X: init_utils.predict_dec(params, X.T), X_train,
np.squeeze(Y_train))
# %% [markdown]
# # no-regularization, L2-regularization, dropout
# %%
X_re_train, Y_re_train, X_re_test, Y_re_test = reg_utils.load_2D_dataset(
is_plot=True)
# %%
def forward_propagate_with_reg(X, params, keep_prob=1):
# retrieve parameters
W1 = params["W1"]
b1 = params["b1"]
