def setUp(self): self.parent = optimizer.Hyperparameter() self.parent.x = 1 self.parent.y = 2 self.child = optimizer.Hyperparameter(self.parent) self.child.y = 3 self.child.z = 4
from __future__ import division import math import numpy from chainer import backend from chainer.backends import cuda from chainer import optimizer _default_hyperparam = optimizer.Hyperparameter() _default_hyperparam.alpha = 0.001 _default_hyperparam.beta1 = 0.9 _default_hyperparam.beta2 = 0.999 _default_hyperparam.eps = 1e-8 _default_hyperparam.eta = 1.0 _default_hyperparam.weight_decay_rate = 0 _default_hyperparam.amsgrad = False def _learning_rate(hp, t): if t == 0: raise RuntimeError( 'Can\'t determine the learning rate of Adam optimizer ' 'because the update steps have not been started.') fix1 = 1. - math.pow(hp.beta1, t) fix2 = 1. - math.pow(hp.beta2, t) return hp.alpha * math.sqrt(fix2) / fix1 class AdamRule(optimizer.UpdateRule): """Update rule of Adam optimization algorithm.
from chainer.backends import intel64 from chainer import optimizer from chainer import types if types.TYPE_CHECKING: import typing_extensions as tpe class SGDHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of vanilla stochastic gradient descent. This is only for PEP 544 compliant static type checkers. """ lr = None # type: float _default_hyperparam = optimizer.Hyperparameter( ) # type: SGDHyperparameter # NOQA _default_hyperparam.lr = 0.01 class SGDRule(optimizer.UpdateRule): """Update rule of vanilla stochastic gradient descent. See :class:`~chainer.optimizers.SGD` for the default values of the hyperparameters. Args: parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter that provides the default values. lr (float): Learning rate. """
if types.TYPE_CHECKING: import typing_extensions as tpe class MSVAGHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of M-SVAG. This is only for PEP 544 compliant static type checkers. """ lr = None # type: float beta = None # type: float eta = None # type: float weight_decay_rate = None # type: float _default_hyperparam = optimizer.Hyperparameter( ) # type: MSVAGHyperparameter # NOQA _default_hyperparam.lr = 0.1 _default_hyperparam.beta = 0.9 _default_hyperparam.eta = 1.0 _default_hyperparam.weight_decay_rate = 0 class MSVAGRule(optimizer.UpdateRule): """Update rule of the M-SVAG optimization algorithm. See: `Dissecting Adam: The Sign, Magnitude and Variance of Stochastic \ Gradients <https://arxiv.org/abs/1705.07774>`_ Modified for proper weight decay. See: `Fixing Weight Decay Regularization in Adam \
from chainer import types if types.TYPE_CHECKING: import typing_extensions as tpe class CorrectedMomentumSGDHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of corrected momentum SGD. This is only for PEP 544 compliant static type checkers. """ lr = None # type: float momentum = None # type: float _default_hyperparam = optimizer.Hyperparameter() # type: CorrectedMomentumSGDHyperparameter # NOQA _default_hyperparam.lr = 0.01 _default_hyperparam.momentum = 0.9 class CorrectedMomentumSGDRule(optimizer.UpdateRule): """Update rule for the corrected momentum SGD. See :class:`~chainer.optimizers.CorrectedMomentumSGD` for the default values of the hyperparameters. Args: parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter that provides the default values. lr (float): Learning rate.
if types.TYPE_CHECKING: import typing_extensions as tpe class RMSpropHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of RMSprop. This is only for PEP 544 compliant static type checkers. """ lr = None # type: float alpha = None # type: float eps = None # type: float eps_inside_sqrt = None # type: bool _default_hyperparam = optimizer.Hyperparameter( ) # type: RMSpropHyperparameter # NOQA _default_hyperparam.lr = 0.01 _default_hyperparam.alpha = 0.99 _default_hyperparam.eps = 1e-8 _default_hyperparam.eps_inside_sqrt = False class RMSpropRule(optimizer.UpdateRule): """Update rule for RMSprop. See :class:`~chainer.optimizers.RMSprop` for the default values of the hyperparameters. Args: parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter that provides the default values.
from chainer import optimizer from chainer import types if types.TYPE_CHECKING: import typing_extensions as tpe class AdaDeltaHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of Zeiler's ADADELTA. This is only for PEP 544 compliant static type checkers. """ rho = None # type: float eps = None # type: float _default_hyperparam = optimizer.Hyperparameter( ) # type: AdaDeltaHyperparameter # NOQA _default_hyperparam.rho = 0.95 _default_hyperparam.eps = 1e-6 class AdaDeltaRule(optimizer.UpdateRule): """Update rule of Zeiler's ADADELTA. See :class:`~chainer.optimizers.AdaDelta` for the default values of the hyperparameters. Args: parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter that provides the default values. rho (float): Exponential decay rate of the first and second order moments.
from chainer import optimizer from chainer import types if types.TYPE_CHECKING: import typing_extensions as tpe class AdaGradHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of AdaGrad. This is only for PEP 544 compliant static type checkers. """ lr = None # type: float eps = None # type: float _default_hyperparam = optimizer.Hyperparameter( ) # type: AdaGradHyperparameter # NOQA _default_hyperparam.lr = 0.001 _default_hyperparam.eps = 1e-8 class AdaGradRule(optimizer.UpdateRule): """Update rule of AdaGrad. See :class:`~chainer.optimizers.AdaGrad` for the default values of the hyperparameters. Args: parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter that provides the default values. lr (float): Learning rate. eps (float): Small value for the numerical stability.
from chainer import optimizer from chainer import types if types.TYPE_CHECKING: import typing_extensions as tpe class NesterovAGHyperparameter(tpe.Protocol): """Protocol class for hyperparameter of Nesterov's Accelerated Gradient. This is only for PEP 544 compliant static type checkers. """ lr = None # type: float momentum = None # type: float _default_hyperparam = optimizer.Hyperparameter( ) # type: NesterovAGHyperparameter # NOQA _default_hyperparam.lr = 0.01 _default_hyperparam.momentum = 0.9 class NesterovAGRule(optimizer.UpdateRule): """Update rule for Nesterov's Accelerated Gradient. See :class:`~chainer.optimizers.NesterovAG` for the default values of the hyperparameters. Args: parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter that provides the default values. lr (float): Learning rate. momentum (float): Exponential decay rate of the first order moment.