'''
features, labels = get_data_ch7()
def init_adadelta_states():
s_w, s_b = torch.zeros((features.shape[1], 1), dtype=torch.float32), torch.zeros(1, dtype=torch.float32)
delta_w, delta_b = torch.zeros((features.shape[1], 1), dtype=torch.float32), torch.zeros(1, dtype=torch.float32)
return ((s_w, delta_w), (s_b, delta_b))
def adadelta(params, states, hyperparams):
rho, eps = hyperparams['rho'], 1e-5
for p, (s, delta) in zip(params, states): # 依次取出 w, b 进行计算
s[:] = rho * s + (1 - rho) * (p.grad.data**2)
# print(s)
g = p.grad.data * torch.sqrt((delta + eps) / (s + eps))
p.data -= g
delta[:] = rho * delta + (1 - rho) * g * g
'''
使用超参数ρ=0.9来训练模型。
'''
# d2l.train_ch7(adadelta, init_adadelta_states(), {'rho': 0.9}, features, labels)
'''
简洁实现:
通过名称为Adadelta的优化器方法，我们便可使用PyTorch提供的AdaDelta算法。它的超参数可以通过rho来指定。
'''
d2l.train_pytorch_ch7(torch.optim.Adadelta, {'rho': 0.9}, features, labels)
'''
AdaDelta算法没有学习率超参数，它通过使用有关自变量更新量平方的指数加权移动平均的项来替代RMSProp算法中的学习率。
'''
s_w, s_b = torch.zeros(
(features.shape[1], 1),
dtype=torch.float32), torch.zeros(1, dtype=torch.float32)
return ((v_w, s_w), (v_b, s_b))
def adam(params, states, hyperparams):
beta1, beta2, eps = 0.9, 0.999, 1e-6
for p, (v, s) in zip(params, states):
v[:] = beta1 * v + (1 - beta1) * p.grad.data
s[:] = beta2 * s + (1 - beta2) * p.grad.data**2
v_bias_corr = v / (1 - beta1**hyperparams['t'])
s_bias_corr = s / (1 - beta2**hyperparams['t'])
p.data -= hyperparams['lr'] * v_bias_corr / (torch.sqrt(s_bias_corr) +
eps)
hyperparams['t'] += 1
'''
使用学习率为0.01的Adam算法来训练模型。
'''
# d2l.train_ch7(adam, init_adam_states(), {'lr': 0.01, 't': 1}, features, labels)
'''
简洁实现:
通过名称为“Adam”的优化器实例，我们便可使用PyTorch提供的Adam算法。
'''
d2l.train_pytorch_ch7(torch.optim.Adam, {'lr': 0.01}, features, labels)
'''
Adam算法在RMSProp算法的基础上对小批量随机梯度也做了指数加权移动平均。
Adam算法使用了偏差修正。
'''
'''
zip()用法
'''
# params = [1, 2, 3]
# states = [5, 6, 7]
# for p, v in zip(params, states):
# print(p)
# print('-'*100)
# print(v)
# 1
# ----------------------------------------------------------------------------------------------------
# 5
# 2
# ----------------------------------------------------------------------------------------------------
# 6
# 3
# ----------------------------------------------------------------------------------------------------
# 7
#
# print('='*100)
# for p in zip(params, states):
# print(p)
# (1, 5)
# (2, 6)
# (3, 7)
'''
简洁实现：
'''
d2l.train_pytorch_ch7(torch.optim.SGD, {'lr': 0.004, 'momentum': 0.9},
features, labels)
import torch
import d2lzh_pytorch as d2l
features, labels = d2l.get_data_ch7()
def init_rmsprop_states():
s_w = torch.zeros((features.shape[1], 1), dtype=torch.float32)
s_b = torch.zeros(1, dtype=torch.float32)
return (s_w, s_b)
def rmsprop(params, states, hyperparams):
gamma, eps = hyperparams['gamma'], 1e-6
for p, s in zip(params, states):
s.data = gamma * s.data + (1 - gamma) * (p.grad.data)**2
p.data -= hyperparams['lr'] * p.grad.data / torch.sqrt(s + eps)
# d2l.train_ch7(rmsprop, init_rmsprop_states(), {'lr': 0.01, 'gamma': 0.9},
# features, labels)
d2l.train_pytorch_ch7(torch.optim.RMSprop, {
'lr': 0.01,
'alpha': 0.9
}, features, labels)