s_hat = s / (1 - beta2 ** t) p.data -= (v_hat * lr) / (torch.sqrt(s_hat) + err) states['t'] = t + 1 if '__main__' == __name__: from learn_pytorch.OptimizationAlgorithm.loss_func import sqrt_loss from learn_pytorch.OptimizationAlgorithm.traner import train from learn_pytorch.OptimizationAlgorithm.net import LinearReg from utility.load_airfoil_self_noise import load_airfoil_self_noise features, labels = load_airfoil_self_noise('../../data/airfoil_self_noise.dat') batch_size = 10 lr = 0.01 num_epoch = 2 net = LinearReg(5, 1) params = net.parameters() states = init_states(params) train(Adam, states, {'lr': lr}, net, sqrt_loss, features, labels, batch_size, num_epoch )
def sgd_momentum(params, states, hyperparams): for p, v in zip(params, states): momentum = hyperparams['momentum'] lr = hyperparams['lr'] v.data = momentum * v.data + lr * p.grad.data p.data -= v.data if '__main__' == __name__: from learn_pytorch.OptimizationAlgorithm.loss_func import sqrt_loss from learn_pytorch.OptimizationAlgorithm.traner import train from learn_pytorch.OptimizationAlgorithm.net import LinearReg from utility.load_airfoil_self_noise import load_airfoil_self_noise features, labels = load_airfoil_self_noise( '../../data/airfoil_self_noise.dat') batch_size = 10 lr = 0.02 num_epoch = 2 net = LinearReg(5, 1) params = net.parameters() states = init_states(len(list(params))) train(sgd_momentum, states, { 'lr': lr, 'momentum': 0.5 }, net, sqrt_loss, features, labels, batch_size, num_epoch)
def sgd(params, states, hyperparams): for param in params: param.data -= hyperparams['lr'] * param.grad.data if '__main__' == __name__: from learn_pytorch.OptimizationAlgorithm.loss_func import sqrt_loss from learn_pytorch.OptimizationAlgorithm.traner import train from learn_pytorch.OptimizationAlgorithm.net import LinearReg from utility.load_airfoil_self_noise import load_airfoil_self_noise features, labels = load_airfoil_self_noise('../../data/airfoil_self_noise.dat') batch_size = 1 lr = 0.005 num_epoch = 2 train(sgd, None, {'lr': lr}, LinearReg(5, 1), sqrt_loss, features, labels, batch_size, num_epoch )
def RMSProp(params, states, hyperparams): lr = hyperparams['lr'] alpha = hyperparams['alpha'] err = 1e-6 for p, s in zip(params, states): s.data += s.data * alpha + (1 - alpha) * (p.grad.data)**2 p.data -= (p.grad.data * lr) / torch.sqrt(s + err) if '__main__' == __name__: from learn_pytorch.OptimizationAlgorithm.loss_func import sqrt_loss from learn_pytorch.OptimizationAlgorithm.traner import train from learn_pytorch.OptimizationAlgorithm.net import LinearReg from utility.load_airfoil_self_noise import load_airfoil_self_noise features, labels = load_airfoil_self_noise( '../../data/airfoil_self_noise.dat') batch_size = 10 lr = 0.01 num_epoch = 2 net = LinearReg(5, 1) params = net.parameters() states = init_states(params) train(RMSProp, states, { 'lr': lr, 'alpha': 0.5 }, net, sqrt_loss, features, labels, batch_size, num_epoch, 'RMSprop')
# g = p.grad.data * torch.sqrt((delta + eps) / (s + eps)) # p.data -= g # delta[:] = rho * delta + (1 - rho) * g * g if '__main__' == __name__: from learn_pytorch.OptimizationAlgorithm.loss_func import sqrt_loss from learn_pytorch.OptimizationAlgorithm.traner import train from learn_pytorch.OptimizationAlgorithm.net import LinearReg from utility.load_airfoil_self_noise import load_airfoil_self_noise features, labels = load_airfoil_self_noise('../../data/airfoil_self_noise.dat') batch_size = 10 lr = 0.1 num_epoch = 2 net = LinearReg(5, 1) params = net.parameters() states = init_states(params) # states = init_adadelta_states() train(adadelta, states, {'rho': 0.99}, net, sqrt_loss, features, labels, batch_size, num_epoch )