def many_loss_graphs(results_losses, graph_draw_num=9, col_num=3, sort=True, smooth=True, verbose=True):
row_num = int(np.ceil(graph_draw_num / col_num))
i = 0
print()
if sort:
losses_train_items = sorted(results_losses.items(), key=lambda x:x[1][-1], reverse=False)
else:
losses_train_items = results_losses.items()
for key, one_losses in losses_train_items:
one_loss = ("%.2f" % one_losses[-1]).rjust(5) ### str().rjust(5)
if verbose: print(f"{'B' if sort else 'T'}est-{str(i+1).ljust(2)} (val acc:{one_loss}%) | " + key)
plt.subplot(row_num, col_num, i+1)
plt.title(key)
plt.ylim(0, 5)
# if i % col_num: plt.yticks([])
if i < graph_draw_num-col_num: plt.xticks([])
x = np.arange(len(one_losses))
if smooth:
plt.plot(x, smooth_curve(one_losses), label="loss")
else:
plt.plot(x, one_losses)
i += 1
if i >= graph_draw_num:
break
plt.show()
def main():
#0:MNISTデータの読み込み
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)
train_size = x_train.shape[0]
batch_size = 128
max_iteraations = 2000
# 1:実験の設定
optimizers = {}
optimizers['SGD'] = SGD()
optimizers['Momentum'] = Momentum()
optimizers['AdaGrad'] = AdaGrad()
optimizers['Adam'] = Adam()
#optimizers['RMSprop'] = RMSprop()
networks = {}
train_loss = {}
for key in optimizers.keys():
networks[key] = MultiLayerNet(
input_size=784, hidden_size_list=[100, 100, 100, 100],
output_size=10)
train_loss[key] = []
# 2:訓練の開始
for i in range(max_iteraations):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
for key in optimizers.keys():
grads = networks[key].gradient(x_batch, t_batch)
optimizers[key].update(networks[key].params, grads)
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
if i % 100 == 0:
print ("==========" + "iteration:" + str(i) + "==========")
for key in optimizers.keys():
loss = networks[key].loss(x_batch, t_batch)
print (key + ":" + str(loss))
# 3.グラフの描画
markers = {"SGD": "o", "Momentum": "x", "AdaGrad": "s", "Adam": "D"}
x = np.arange(max_iteraations)
for key in optimizers.keys():
plt.plot(x, smooth_curve(train_loss[key]), marker=markers[key], markevery=100, label=key)
plt.xlabel=("iterations")
plt.ylabel=("loss")
plt.ylim(0, 1)
plt.legend()
plt.show()
def main():
x_train, t_train, x_test, x_test = get_data()
# 実験の設定
train_size = x_train.shape[0]
batch_size = 128
max_iterations = 2000
weight_init_types = {'std=0.01': 0.01, 'Xavier': 'sigmoid', 'He': 'relu'}
optimizer = SGD(lr=0.01)
networks = {}
train_loss = {}
for key, weight_type in weight_init_types.items():
networks[key] = MultiLayerNet(input_size=784, hidden_size_list=[100, 100, 100, 100],
output_size=10, weight_init_std=weight_type)
train_loss[key] = []
# 訓練の開始
for i in range(max_iterations):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
# 実験のパラメータごとに実行
for key in weight_init_types.keys():
# 勾配を計算
grads = networks[key].gradient(x_batch, t_batch)
# パラメータを更新
optimizer.update(networks[key].params, grads)
# 更新されたパラメータで改めて損失を算出
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
# terminal check
if i % 100 == 0:
print(f'=========iteration:{str(i)}===========')
for key in weight_init_types.keys():
loss = networks[key].loss(x_batch, t_batch)
print(f'{key}:{str(loss)}')
# グラフの描画
fig = plt.figure()
markers = {'std=0.01': 'o', 'Xavier': 's', 'He': 'D'}
x = np.arange(max_iterations)
for key in weight_init_types.keys():
plt.plot(x, smooth_curve(train_loss[key]), marker=markers[key], markevery=100, label=key)
plt.xlabel('iterations')
plt.ylabel('loss')
plt.ylim(0, 2.5)
plt.legend()
fig.savefig('../images/weight_init_compare.png')
def loss_graphs(train_losses, str_optims, smooth=True, colors=('b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'), ylim=2.5):
x = np.arange(len(train_losses[str_optims[0]]))
max_y = 0
for str_optim, color in zip(str_optims, colors):
y = smooth_curve(train_losses[str_optim]) if smooth else train_losses[str_optim]
plt.plot(x, y, label=str_optim) # f"-{color}"
max_y = np.array(train_losses[str_optim]).max() if max_y > train_losses[str_optim][-1] else max_y
plt.xlabel("iterations")
plt.ylabel("loss")
ylim = math.ceil(max_y*10)/10
plt.ylim(0, ylim)
plt.legend(loc='upper right')
def loss_graph(train_losses, smooth=True):
x = np.arange(len(train_losses))
if smooth:
plt.plot(x, smooth_curve(train_losses), f"-", label="loss")
else:
plt.plot(x, train_losses, f"-", label="loss")
plt.xlabel("iterations")
plt.ylabel("loss")
ylim = math.ceil(np.array(train_losses).max()*10)/10
plt.ylim(0, ylim)
plt.legend(loc='upper right')
plt.show()
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
for key in optimizers.keys():
grads = networks[key].gradient(x_batch, t_batch)
optimizers[key].update(networks[key].params, grads)
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
if i % 100 == 0:
print(f"========= iteration: {i} ==========")
for key in optimizers.keys():
loss = networks[key].loss(x_batch, t_batch)
print(f"{key} : {loss}")
# 3. 그래프 그리기
markers = {"SGD": "o", "Momentum": "x", "AdaGrad": "s", "Adam": "D"}
x = np.arange(max_iterations)
plt.figure(figsize=(8, 6))
for key in optimizers.keys():
plt.plot(x,
smooth_curve(train_loss[key]),
marker=markers[key],
markevery=100,
label=key)
plt.xlabel("iterations")
plt.ylabel("loss")
plt.ylim(0, 1)
plt.legend()
plt.show()
# 2. 훈련 시작==========
for i in range(max_iterations):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
for key in optimizers.keys():
grads = networks[key].gradient(x_batch, t_batch)
optimizers[key].update(networks[key].params, grads)
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
if i % 100 == 0:
print( "===========" + "iteration:" + str(i) + "===========")
for key in optimizers.keys():
loss = networks[key].loss(x_batch, t_batch)
print(key + ":" + str(loss))
# 3. 그래프 그리기==========
markers = {"SGD": "o", "Momentum": "x", "AdaGrad": "s", "Adam": "D"}
x = np.arange(max_iterations)
for key in optimizers.keys():
plt.plot(x, smooth_curve(train_loss[key]), marker=markers[key], markevery=100, label=key)
plt.xlabel("iterations")
plt.ylabel("loss")
plt.ylim(0, 1)
plt.legend()
plt.show()
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
for key in optimizers.keys():
grads = networks[key].gradient(x_batch, t_batch)
optimizers[key].update(networks[key].params, grads)
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
if i % 100 == 0:
print("===========" + "iteration:" + str(i) + "===========")
for key in optimizers.keys():
loss = networks[key].loss(x_batch, t_batch)
print(key + ":" + str(loss))
for key in optimizers.keys():
accuracy = networks[key].accuracy(x_test, t_test)
test_accuracy[key].append(accuracy)
print(test_accuracy)
# 3.グラフの描画==========
x = np.arange(max_iterations)
for key in optimizers.keys():
plt.plot(x, smooth_curve(train_loss[key]), label=key)
plt.xlabel("iterations")
plt.ylabel("loss")
plt.ylim(0, 1)
plt.legend()
plt.show()
# 2. 훈련 시작==========
for i in range(max_iterations):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
for key in weight_init_types.keys():
grads = networks[key].gradient(x_batch, t_batch)
optimizer.update(networks[key].params, grads)
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
if i % 100 == 0:
print("===========" + "iteration:" + str(i) + "===========")
for key in weight_init_types.keys():
loss = networks[key].loss(x_batch, t_batch)
print(key + ":" + str(loss))
# 3. 그래프 그리기==========
markers = {'std=0.01': 'o', 'Xavier': 's', 'He': 'D'}
x = np.arange(max_iterations)
for key in weight_init_types.keys():
plt.plot(x, smooth_curve(train_loss[key]), marker=markers[key], markevery=100, label=key)
plt.xlabel("iterations")
plt.ylabel("loss")
plt.ylim(0, 2.5)
plt.legend()
plt.show()
def main():
(train_x, train_label), _ = load_mnist()
train_size = train_x.shape[0]
batch_size = 128
max_iterations = 2000
# 打乱训练集顺序，每次训练随机获取一个batch大小的训练样本
batch_mask = np.arange(train_size)
np.random.shuffle(batch_mask)
# 将要对比的优化器
optimizers = {}
optimizers['SGD'] = SGD()
optimizers['Momentum'] = Momentum()
optimizers['AdaGrad'] = AdaGrad()
optimizers['Adam'] = Adam()
optimizers['RMSProp'] = RMSProp()
# 为每个优化器生成一个五层全连接神经网络
networks = {}
train_loss_list = {}
for key in optimizers.keys():
networks[key] = MultiLayerNet(input_size=784,
hidden_size_list=[100, 100, 100, 100],
output_size=10)
train_loss_list[key] = [] # 每个优化器都记录在训练集上的损失值
left = 0
for i in range(max_iterations):
# 获取一个batch
batch_x, batch_label, left = get_batch(train_x, train_label,
batch_mask, batch_size, left)
# 计算梯度，然后用不同优化器去更新参数
# 记录每次更新后的损失值
for key in optimizers.keys():
grads = networks[key].gradient(batch_x, batch_label)
optimizers[key].update(networks[key].params, grads)
loss = networks[key].loss(batch_x, batch_label)
train_loss_list[key].append(loss)
# 每迭代100次就输出一次当前各优化器的损失值
if i % 100 == 0:
print("=" * 15 + "iteration: " + str(i) + "=" * 15)
for key in optimizers.keys():
loss = train_loss_list[key][-1]
print(key + ": " + str(loss))
# 绘制损失值随迭代次数变化图
markers = {
'SGD': 'o',
'Momentum': 'x',
'AdaGrad': 's',
'Adam': 'D',
'RMSProp': 'v'
}
x = np.arange(max_iterations)
for key in optimizers.keys():
plt.plot(x,
smooth_curve(train_loss_list[key]),
marker=markers[key],
markevery=100,
label=key)
plt.xlabel('iterations')
plt.ylabel('loss')
plt.ylim(0, 1)
plt.legend()
plt.show()
def main():
# MNISTデータの読み込み
x_train, t_train, x_test, t_test = get_data()
# 実験の設定
train_size = x_train.shape[0]
batch_size = 128
max_iterations = 2000
optimizers = {}
optimizers['SGD'] = SGD()
optimizers['Momentum'] = Momentum()
optimizers['AdaGrad'] = AdaGrad()
optimizers['Adam'] = Adam()
optimizers['RMSprop'] = RMSprop()
networks = {}
train_loss = {}
for key in optimizers.keys():
networks[key] = MultiLayerNet(input_size=784,
hidden_size_list=[100, 100, 100, 100],
output_size=10)
train_loss[key] = []
# 訓練の開始
for i in range(max_iterations):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
for key in optimizers.keys():
grads = networks[key].gradient(x_batch, t_batch)
optimizers[key].update(networks[key].params,
grads) # 現在のパラメータとその勾配を渡す
loss = networks[key].loss(x_batch, t_batch)
train_loss[key].append(loss)
# terminal check
if i % 100 == 0:
print(f'=========iteration:{str(i)}===========')
for key in optimizers.keys():
loss = networks[key].loss(x_batch, t_batch)
print(f'{key}:{str(loss)}')
# グラフの描画
fig = plt.figure()
markers = {
'SGD': 'o',
'Momentum': 'x',
'AdaGrad': 's',
'Adam': 'D',
'RMSprop': '+'
}
x = np.arange(max_iterations)
for key in optimizers.keys():
plt.plot(x,
smooth_curve(train_loss[key]),
marker=markers[key],
markevery=100,
label=key)
plt.xlabel('iterations')
plt.ylabel('loss')
plt.ylim(0, 1)
plt.legend()
fig.savefig('../images/optimizer_compare.png')
def main():
(train_x, train_label), _ = load_mnist()
train_size = train_x.shape[0]
batch_size = 128
max_iterations = 2000
# 打乱训练集顺序，每次训练随机获取一个batch大小的训练样本
batch_mask = np.arange(train_size)
np.random.shuffle(batch_mask)
# 使用SGD优化器
optimizer = SGD(lr=0.01)
# 将要对比的权重初始化方式： 0.01， Xavier， He 三种
weight_init_types = {'std=0.01': 0.01, 'Xavier': 'sigmoid', 'He': 'relu'}
# 为每个优化器生成一个五层全连接神经网络
networks = {}
train_loss_list = {}
for key, weight_init_type in weight_init_types.items():
networks[key] = MultiLayerNet(input_size=784,
hidden_size_list=[100, 100, 100, 100],
output_size=10,
weight_init_std=weight_init_type)
train_loss_list[key] = [] # 每个优化器都记录在训练集上的损失值
left = 0
for i in range(max_iterations):
# 获取一个batch
batch_x, batch_label, left = get_batch(train_x, train_label,
batch_mask, batch_size, left)
# 计算梯度，然后用不同优化器去更新参数
# 记录每次更新后的损失值
for key in weight_init_types.keys():
grads = networks[key].gradient(batch_x, batch_label)
optimizer.update(networks[key].params, grads)
loss = networks[key].loss(batch_x, batch_label)
train_loss_list[key].append(loss)
# 每迭代100次就输出一次当前各优化器的损失值
if i % 100 == 0:
print("=" * 15 + "iteration: " + str(i) + "=" * 15)
for key in weight_init_types.keys():
loss = train_loss_list[key][-1]
print(key + ": " + str(loss))
# 绘制损失值随迭代次数变化图
markers = {'std=0.01': 'o', 'Xavier': 's', 'He': 'D'}
x = np.arange(max_iterations)
for key in weight_init_types.keys():
plt.plot(x,
smooth_curve(train_loss_list[key]),
marker=markers[key],
markevery=100,
label=key)
plt.xlabel("iterations")
plt.ylabel("loss")
plt.ylim(0, 2.5)
plt.legend()
plt.show()
loss[key] = []
# 2:訓練の開始==========
for i in range(iter_num):
# 60000サンプルから128サンプルをランダムに選ぶ
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
#
for key in weight_type.keys():
# 勾配を求めて重みを更新する
grads = network[key].gradient(x_batch, t_batch)
optimizer.update(network[key].params, grads)
loss[key].append(network[key].loss(x_batch, t_batch))
if i % 100 == 0:
print("--iter_num = {}--".format(i))
for key in weight_type.keys():
print("{} loss : {}".format(key, loss[key][-1]))
# 3.グラフの描画==========
for key in weight_type.keys():
plt.plot(np.arange(iter_num), smooth_curve(loss[key]), label=key)
plt.xlabel("iterations")
plt.ylabel("loss")
plt.legend()
plt.show()
losses = {}
for key in keys:
networks[key] = TwoLayerNet(784, 50, 10)
losses[key] = []
iter_count = 10000
batch_size = 100
train_size = x_train.shape[0]
epoch_size = train_size / batch_size
for i in range(iter_count):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
grads = {}
for key in keys:
optimizers[key].update(networks[key].params,
networks[key].gradient(x_batch, t_batch))
losses[key].append(networks[key].loss(x_batch, t_batch))
if i % epoch_size == 0:
print("====================iter_count : " + str(i))
for key in keys:
accloss = networks[key].acc(x_test, t_test)
print(str(key) + " : " + str(accloss))
for key in keys:
plt.plot(range(iter_count), smooth_curve(losses[key]), markevery=100)
plt.ylim(0, 1)
plt.show()