def test_learner_init():
i = C.input_variable(shape=(1,), needs_gradient=True, name='a')
w = parameter(shape=(1,))
res = i * w
learner = sgd(res.parameters, lr=learning_rate_schedule(0.1, UnitType.sample))
assert learner.learning_rate() == 0.1
learner.reset_learning_rate(learning_rate_schedule([1,2,3], UnitType.minibatch));
assert learner.learning_rate() == 1.0
learner_parameter = learner.parameters
from cntk.variables import Parameter
param = learner_parameter[0]
assert isinstance(param, Parameter)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
momentum_time_constant = C.momentum_as_time_constant_schedule(1100)
lr_per_sample = learning_rate_schedule(0.1, UnitType.sample)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant, unit_gain_value)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant, unit_gain=unit_gain_value)
C.set_default_unit_gain_value(False)
unit_gain_value = C.default_unit_gain_value()
assert not unit_gain_value
lr_per_sample = learning_rate_schedule([0.1, 0.2], UnitType.sample)
C.nesterov(res.parameters, lr=lr_per_sample, momentum=momentum_time_constant)
C.nesterov(res.parameters, lr_per_sample, momentum_time_constant, unit_gain_value)
C.nesterov(res.parameters, lr=lr_per_sample, momentum=momentum_time_constant, unit_gain=unit_gain_value)
lr_per_sample = learning_rate_schedule([0.1]*3 +[0.2]*2 +[0.3], UnitType.sample)
C.adagrad(res.parameters, lr=lr_per_sample, need_ave_multiplier=True)
C.set_default_unit_gain_value(True)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
lr_per_sample = learning_rate_schedule([(3,0.1), (2, 0.2), (1, 0.3)], UnitType.sample)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum_time_constant)
C.fsadagrad(res.parameters, lr_per_sample, momentum_time_constant, unit_gain_value)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum_time_constant, unit_gain=unit_gain_value)
gamma, inc, dec, max, min = [0.1]*5
lr_per_sample = learning_rate_schedule([0.1, 0.2], UnitType.sample, 100)
C.rmsprop(res.parameters, lr_per_sample, gamma, inc, dec, max, min, True)
C.set_default_use_mean_gradient_value(False)
use_mean_gradient_value = C.default_use_mean_gradient_value()
assert not use_mean_gradient_value
C.adadelta(res.parameters, lr_per_sample)
C.set_default_use_mean_gradient_value(True)
use_mean_gradient_value = C.default_use_mean_gradient_value()
assert use_mean_gradient_value
C.adadelta(res.parameters, lr_per_sample)
def test_learner_init():
i = C.input_variable(shape=(1, ), needs_gradient=True, name='a')
w = parameter(shape=(1, ))
res = i * w
#test new API: learning_parameter_schedule
#explictly specify reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=0.1, minibatch_size=25)
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == 25 #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 25
assert learner.learning_rate() == 0.1
#no explictly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1))
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
learner = sgd(res.parameters,
lr=learning_parameter_schedule(0.1, 20),
minibatch_size=25)
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == 25 #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 20
assert learner.learning_rate() == 0.1
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1, 20))
assert learner.is_compatible_mode() == False
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 20
assert learner.learning_rate() == 0.1
#no explictly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1))
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
#no explictly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters,
lr=learning_parameter_schedule(0.1),
minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
learner = sgd(res.parameters,
lr=learning_parameter_schedule(0.1, 20),
minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 20
assert learner.learning_rate() == 0.1
#no explictly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters,
lr=learning_parameter_schedule(0.1),
minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
mysgd = C.sgd(parameters=res.parameters, lr=0.4, minibatch_size=32)
assert mysgd.minibatch_size == 32
assert mysgd._learning_rate_schedule.minibatch_size == 32
assert mysgd.learning_rate() == 0.4
mymomentum = C.momentum_sgd(parameters=res.parameters,
lr=0.4,
momentum=0.9,
minibatch_size=32)
assert mymomentum.minibatch_size == 32
assert mymomentum._learning_rate_schedule.minibatch_size == 32
assert mymomentum.learning_rate() == 0.4
myadadelta = C.adadelta(parameters=res.parameters,
lr=0.4,
minibatch_size=32)
assert myadadelta.minibatch_size == 32
assert myadadelta._learning_rate_schedule.minibatch_size == 32
assert myadadelta.learning_rate() == 0.4
myadam = C.adam(parameters=res.parameters,
lr=0.4,
momentum=0.9,
variance_momentum=0.9,
minibatch_size=32)
assert myadam.minibatch_size == 32
assert myadam._learning_rate_schedule.minibatch_size == 32
assert myadam.learning_rate() == 0.4
myadagrad = C.adagrad(parameters=res.parameters, lr=0.4, minibatch_size=32)
assert myadagrad.minibatch_size == 32
assert myadagrad._learning_rate_schedule.minibatch_size == 32
assert myadagrad.learning_rate() == 0.4
myfsadagrad = C.fsadagrad(parameters=res.parameters,
lr=0.4,
momentum=0.9,
variance_momentum=0.9,
minibatch_size=32)
assert myfsadagrad.minibatch_size == 32
assert myfsadagrad._learning_rate_schedule.minibatch_size == 32
assert myfsadagrad.learning_rate() == 0.4
mynesterov = C.nesterov(parameters=res.parameters,
lr=0.4,
momentum=0.9,
minibatch_size=32)
assert mynesterov.minibatch_size == 32
assert mynesterov._learning_rate_schedule.minibatch_size == 32
assert mynesterov.learning_rate() == 0.4
myrmsrop = C.rmsprop(parameters=res.parameters,
lr=0.4,
gamma=0.5,
inc=1.2,
dec=0.7,
max=10,
min=1e-8,
minibatch_size=32)
assert myrmsrop.minibatch_size == 32
assert myrmsrop._learning_rate_schedule.minibatch_size == 32
assert myrmsrop.learning_rate() == 0.4
mysgd = C.sgd(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
minibatch_size=32,
epoch_size=512)
assert mysgd.minibatch_size == 32
assert mysgd._learning_rate_schedule.minibatch_size == 32
assert mysgd._learning_rate_schedule[0] == 0.4
assert mysgd._learning_rate_schedule[512] == 0.1
assert mysgd._learning_rate_schedule[512 * 2] == 0.001
mymomentum = C.momentum_sgd(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
momentum=[0.9],
minibatch_size=32,
epoch_size=512)
assert mymomentum.minibatch_size == 32
assert mymomentum._learning_rate_schedule.minibatch_size == 32
assert mymomentum._learning_rate_schedule[0] == 0.4
assert mymomentum._learning_rate_schedule[512] == 0.1
assert mymomentum._learning_rate_schedule[512 * 2] == 0.001
myadadelta = C.adadelta(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
minibatch_size=32,
epoch_size=512)
assert myadadelta.minibatch_size == 32
assert myadadelta._learning_rate_schedule.minibatch_size == 32
assert myadadelta._learning_rate_schedule[0] == 0.4
assert myadadelta._learning_rate_schedule[512] == 0.1
assert myadadelta._learning_rate_schedule[512 * 2] == 0.001
myadam = C.adam(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
momentum=[0.9, 0.1, 0.001],
variance_momentum=[0.9],
minibatch_size=32,
epoch_size=512)
assert myadam.minibatch_size == 32
assert myadam._learning_rate_schedule.minibatch_size == 32
assert myadam._learning_rate_schedule[0] == 0.4
assert myadam._learning_rate_schedule[512] == 0.1
assert myadam._learning_rate_schedule[512 * 2] == 0.001
myadagrad = C.adagrad(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
minibatch_size=32,
epoch_size=512)
assert myadagrad.minibatch_size == 32
assert myadagrad._learning_rate_schedule.minibatch_size == 32
assert myadagrad._learning_rate_schedule[0] == 0.4
assert myadagrad._learning_rate_schedule[512] == 0.1
assert myadagrad._learning_rate_schedule[512 * 2] == 0.001
myfsadagrad = C.fsadagrad(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
momentum=[0.9],
variance_momentum=[0.9],
minibatch_size=32,
epoch_size=512)
assert myadagrad.minibatch_size == 32
assert myadagrad._learning_rate_schedule.minibatch_size == 32
assert myadagrad._learning_rate_schedule[0] == 0.4
assert myadagrad._learning_rate_schedule[512] == 0.1
assert myadagrad._learning_rate_schedule[512 * 2] == 0.001
mynesterov = C.nesterov(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
momentum=[0.9],
minibatch_size=32,
epoch_size=512)
assert mynesterov.minibatch_size == 32
assert mynesterov._learning_rate_schedule.minibatch_size == 32
assert mynesterov._learning_rate_schedule[0] == 0.4
assert mynesterov._learning_rate_schedule[512] == 0.1
assert mynesterov._learning_rate_schedule[512 * 2] == 0.001
myrmsrop = C.rmsprop(parameters=res.parameters,
lr=[0.4, 0.1, 0.001],
gamma=0.5,
inc=1.2,
dec=0.7,
max=10,
min=1e-8,
minibatch_size=32,
epoch_size=512)
assert myrmsrop.minibatch_size == 32
assert myrmsrop._learning_rate_schedule.minibatch_size == 32
assert myrmsrop._learning_rate_schedule[0] == 0.4
assert myrmsrop._learning_rate_schedule[512] == 0.1
assert myrmsrop._learning_rate_schedule[512 * 2] == 0.001
learner_parameter = learner.parameters
from cntk.variables import Parameter
param = learner_parameter[0]
assert isinstance(param, Parameter)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
momentum = C.momentum_schedule(0.999, minibatch_size=1)
lr_per_sample = learning_parameter_schedule(0.1, minibatch_size=1)
C.momentum_sgd(res.parameters, lr_per_sample, momentum)
C.momentum_sgd(res.parameters, lr_per_sample, momentum, unit_gain_value)
C.momentum_sgd(res.parameters,
lr_per_sample,
momentum,
unit_gain=unit_gain_value)
C.set_default_unit_gain_value(False)
unit_gain_value = C.default_unit_gain_value()
assert not unit_gain_value
lr_per_sample = learning_parameter_schedule([0.1, 0.2], minibatch_size=1)
C.nesterov(res.parameters, lr=lr_per_sample, momentum=momentum)
C.nesterov(res.parameters, lr_per_sample, momentum, unit_gain_value)
C.nesterov(res.parameters,
lr=lr_per_sample,
momentum=momentum,
unit_gain=unit_gain_value)
lr_per_sample = learning_parameter_schedule([0.1] * 3 + [0.2] * 2 + [0.3],
minibatch_size=1)
C.adagrad(res.parameters, lr=lr_per_sample, need_ave_multiplier=True)
C.set_default_unit_gain_value(True)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
lr_per_sample = learning_parameter_schedule([(3, 0.1), (2, 0.2), (1, 0.3)],
minibatch_size=1)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum)
C.fsadagrad(res.parameters, lr_per_sample, momentum, unit_gain_value)
C.fsadagrad(res.parameters,
lr=lr_per_sample,
momentum=momentum,
unit_gain=unit_gain_value)
gamma, inc, dec, max, min = [0.5, 1.2, 0.7, 10, 1e-8]
lr_per_sample = learning_parameter_schedule([0.1, 0.2],
minibatch_size=1,
epoch_size=100)
C.rmsprop(res.parameters, lr_per_sample, gamma, inc, dec, max, min, True)
C.adadelta(res.parameters, lr_per_sample)
def test_learner_init():
i = C.input_variable(shape=(1,), needs_gradient=True, name='a')
w = parameter(shape=(1,))
res = i * w
#test new API: learning_parameter_schedule
#explicitly specify reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=0.1, minibatch_size = 25)
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == 25 #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 25
assert learner.learning_rate() == 0.1
#no explicitly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1))
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1, 20), minibatch_size = 25)
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == 25 #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 20
assert learner.learning_rate() == 0.1
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1, 20))
assert learner.is_compatible_mode() == False
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 20
assert learner.learning_rate() == 0.1
#no explicitly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1))
assert learner.is_compatible_mode() == False
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
#no explicitly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1), minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1, 20), minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == 20
assert learner.learning_rate() == 0.1
#no explicitly specification of reference minibatch size and learning rate is in number:
learner = sgd(res.parameters, lr=learning_parameter_schedule(0.1), minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.minibatch_size == C.learners.IGNORE #the learner's reference minibatch
#with direct learner learning rate number specification, the learning rate schedule get the reference minibatch size from the learner parameters:
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.learning_rate() == 0.1
mysgd = C.sgd(parameters=res.parameters, lr=0.4, minibatch_size=32)
assert mysgd.minibatch_size == 32
assert mysgd._learning_rate_schedule.minibatch_size == 32
assert mysgd.learning_rate() == 0.4
mymomentum = C.momentum_sgd(parameters=res.parameters, lr=0.4, momentum=0.9, minibatch_size=32)
assert mymomentum.minibatch_size == 32
assert mymomentum._learning_rate_schedule.minibatch_size == 32
assert mymomentum.learning_rate() == 0.4
myadadelta = C.adadelta(parameters=res.parameters, lr=0.4, minibatch_size=32)
assert myadadelta.minibatch_size == 32
assert myadadelta._learning_rate_schedule.minibatch_size == 32
assert myadadelta.learning_rate() == 0.4
myadam = C.adam(parameters=res.parameters, lr=0.4, momentum=0.9, variance_momentum=0.9, minibatch_size=32)
assert myadam.minibatch_size == 32
assert myadam._learning_rate_schedule.minibatch_size == 32
assert myadam.learning_rate() == 0.4
myadagrad = C.adagrad(parameters=res.parameters, lr=0.4, minibatch_size=32)
assert myadagrad.minibatch_size == 32
assert myadagrad._learning_rate_schedule.minibatch_size == 32
assert myadagrad.learning_rate() == 0.4
myfsadagrad = C.fsadagrad(parameters=res.parameters, lr=0.4, momentum=0.9, variance_momentum=0.9,
minibatch_size=32)
assert myfsadagrad.minibatch_size == 32
assert myfsadagrad._learning_rate_schedule.minibatch_size == 32
assert myfsadagrad.learning_rate() == 0.4
mynesterov = C.nesterov(parameters=res.parameters, lr=0.4, momentum=0.9, minibatch_size=32)
assert mynesterov.minibatch_size == 32
assert mynesterov._learning_rate_schedule.minibatch_size == 32
assert mynesterov.learning_rate() == 0.4
myrmsrop = C.rmsprop(parameters=res.parameters, lr=0.4, gamma=0.5, inc=1.2, dec=0.7, max=10, min=1e-8,
minibatch_size=32)
assert myrmsrop.minibatch_size == 32
assert myrmsrop._learning_rate_schedule.minibatch_size == 32
assert myrmsrop.learning_rate() == 0.4
mysgd = C.sgd(parameters=res.parameters, lr=[0.4, 0.1, 0.001], minibatch_size=32, epoch_size=512)
assert mysgd.minibatch_size == 32
assert mysgd._learning_rate_schedule.minibatch_size == 32
assert mysgd._learning_rate_schedule[0] == 0.4
assert mysgd._learning_rate_schedule[512] == 0.1
assert mysgd._learning_rate_schedule[512 * 2] == 0.001
mymomentum = C.momentum_sgd(parameters=res.parameters, lr=[0.4, 0.1, 0.001], momentum=[0.9],
minibatch_size=32, epoch_size=512)
assert mymomentum.minibatch_size == 32
assert mymomentum._learning_rate_schedule.minibatch_size == 32
assert mymomentum._learning_rate_schedule[0] == 0.4
assert mymomentum._learning_rate_schedule[512] == 0.1
assert mymomentum._learning_rate_schedule[512 * 2] == 0.001
myadadelta = C.adadelta(parameters=res.parameters, lr=[0.4, 0.1, 0.001],
minibatch_size=32, epoch_size=512)
assert myadadelta.minibatch_size == 32
assert myadadelta._learning_rate_schedule.minibatch_size == 32
assert myadadelta._learning_rate_schedule[0] == 0.4
assert myadadelta._learning_rate_schedule[512] == 0.1
assert myadadelta._learning_rate_schedule[512 * 2] == 0.001
myadam = C.adam(parameters=res.parameters, lr=[0.4, 0.1, 0.001], momentum=[0.9, 0.1, 0.001], variance_momentum=[0.9],
minibatch_size=32, epoch_size=512)
assert myadam.minibatch_size == 32
assert myadam._learning_rate_schedule.minibatch_size == 32
assert myadam._learning_rate_schedule[0] == 0.4
assert myadam._learning_rate_schedule[512] == 0.1
assert myadam._learning_rate_schedule[512 * 2] == 0.001
myadagrad = C.adagrad(parameters=res.parameters, lr=[0.4, 0.1, 0.001], minibatch_size=32, epoch_size=512)
assert myadagrad.minibatch_size == 32
assert myadagrad._learning_rate_schedule.minibatch_size == 32
assert myadagrad._learning_rate_schedule[0] == 0.4
assert myadagrad._learning_rate_schedule[512] == 0.1
assert myadagrad._learning_rate_schedule[512 * 2] == 0.001
myfsadagrad = C.fsadagrad(parameters=res.parameters, lr=[0.4, 0.1, 0.001], momentum=[0.9],
variance_momentum=[0.9],
minibatch_size=32, epoch_size=512)
assert myadagrad.minibatch_size == 32
assert myadagrad._learning_rate_schedule.minibatch_size == 32
assert myadagrad._learning_rate_schedule[0] == 0.4
assert myadagrad._learning_rate_schedule[512] == 0.1
assert myadagrad._learning_rate_schedule[512 * 2] == 0.001
mynesterov = C.nesterov(parameters=res.parameters, lr=[0.4, 0.1, 0.001], momentum=[0.9],
minibatch_size=32, epoch_size=512)
assert mynesterov.minibatch_size == 32
assert mynesterov._learning_rate_schedule.minibatch_size == 32
assert mynesterov._learning_rate_schedule[0] == 0.4
assert mynesterov._learning_rate_schedule[512] == 0.1
assert mynesterov._learning_rate_schedule[512 * 2] == 0.001
myrmsrop = C.rmsprop(parameters=res.parameters, lr=[0.4, 0.1, 0.001], gamma=0.5, inc=1.2, dec=0.7, max=10,
min=1e-8,
minibatch_size=32, epoch_size=512)
assert myrmsrop.minibatch_size == 32
assert myrmsrop._learning_rate_schedule.minibatch_size == 32
assert myrmsrop._learning_rate_schedule[0] == 0.4
assert myrmsrop._learning_rate_schedule[512] == 0.1
assert myrmsrop._learning_rate_schedule[512 * 2] == 0.001
learner_parameter = learner.parameters
from cntk.variables import Parameter
param = learner_parameter[0]
assert isinstance(param, Parameter)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
momentum = C.momentum_schedule(0.999, minibatch_size=1)
lr_per_sample = learning_parameter_schedule(0.1, minibatch_size = 1)
C.momentum_sgd(res.parameters, lr_per_sample, momentum)
C.momentum_sgd(res.parameters, lr_per_sample, momentum, unit_gain_value)
C.momentum_sgd(res.parameters, lr_per_sample, momentum, unit_gain=unit_gain_value)
C.set_default_unit_gain_value(False)
unit_gain_value = C.default_unit_gain_value()
assert not unit_gain_value
lr_per_sample = learning_parameter_schedule([0.1, 0.2], minibatch_size = 1)
C.nesterov(res.parameters, lr=lr_per_sample, momentum=momentum)
C.nesterov(res.parameters, lr_per_sample, momentum, unit_gain_value)
C.nesterov(res.parameters, lr=lr_per_sample, momentum=momentum, unit_gain=unit_gain_value)
lr_per_sample = learning_parameter_schedule([0.1]*3 +[0.2]*2 +[0.3], minibatch_size=1)
C.adagrad(res.parameters, lr=lr_per_sample, need_ave_multiplier=True)
C.set_default_unit_gain_value(True)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
lr_per_sample = learning_parameter_schedule([(3,0.1), (2, 0.2), (1, 0.3)], minibatch_size=1)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum)
C.fsadagrad(res.parameters, lr_per_sample, momentum, unit_gain_value)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum, unit_gain=unit_gain_value)
gamma, inc, dec, max, min = [0.5, 1.2, 0.7, 10, 1e-8]
lr_per_sample = learning_parameter_schedule([0.1, 0.2], minibatch_size = 1, epoch_size = 100)
C.rmsprop(res.parameters, lr_per_sample, gamma, inc, dec, max, min, True)
C.adadelta(res.parameters, lr_per_sample)
def test_learner_init_legacy():
i = C.input_variable(shape=(1, ), needs_gradient=True, name='a')
w = parameter(shape=(1, ))
res = i * w
# for backcompatibility test
# this will be deprecated in future version
learner = sgd(res.parameters,
lr=learning_rate_schedule(0.1, UnitType.sample))
assert learner._learning_rate_schedule.minibatch_size == 1 # the deprecated per sample schedule should not use compatible mode
assert learner.learning_rate() == 0.1
# for backcompatibility test
# this will be deprecated in future version
# The UnitType will provide per minibatch instruction for the learner
# this will be deprecated in future version
learner = sgd(res.parameters,
lr=learning_rate_schedule(0.1, UnitType.minibatch))
assert learner.is_compatible_mode() == False
assert learner.learning_rate() == 0.1
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == 0
# for backcompatibility test, in reset learning rate, the learner won't receive the reference minibatch size from the schedule
# user will need to specify the reference minibatch size explicitly
# this will be deprecated in future version
learner = sgd(res.parameters, lr=0.1)
learner.reset_learning_rate(
learning_rate_schedule([1, 2, 3], UnitType.minibatch))
assert learner.learning_rate() == 1.0
learner.minibatch_size = C.learners.IGNORE # reset to be per minibatch
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.is_compatible_mode() == True
# for backcompatibility test
# this will be deprecated in future version
learner = sgd(res.parameters,
lr=learning_rate_schedule(0.1, UnitType.sample),
minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.learning_rate() == 0.1
assert learner.minibatch_size == C.learners.IGNORE # the learner's reference minibatch size is still 0
# this will be deprecated in future version: This is logical invalid combination but it was the only way to use mean gradient and set learning rate in the past.
learner = sgd(res.parameters,
lr=learning_rate_schedule(0.1, UnitType.sample),
use_mean_gradient=True)
assert learner.is_compatible_mode() == True
assert learner.learning_rate() == 0.1
#test the override in the new version
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.minibatch_size == C.learners.IGNORE # the learner's reference minibatch size is still 0
# for backcompatibility test
# this will be deprecated in future version
# The UnitType will provide per minibatch instruction for the learner
# this will be deprecated in future version
learner = sgd(res.parameters,
lr=learning_rate_schedule(0.1, UnitType.minibatch),
minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.learning_rate() == 0.1
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
# for backcompatibility test, in reset learning rate, the learner won't receive the reference minibatch size from the schedule
# user will need to specify the reference minibatch size explicitly
# this will be deprecated in future version
learner = sgd(res.parameters, lr=0.1)
learner.reset_learning_rate(
learning_rate_schedule([1, 2, 3], UnitType.minibatch))
assert learner.learning_rate() == 1.0
learner.minibatch_size = C.learners.IGNORE # reset to be per minibatch
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.is_compatible_mode() == True
learner_parameter = learner.parameters
from cntk.variables import Parameter
param = learner_parameter[0]
assert isinstance(param, Parameter)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
# back compatible API test
momentum_time_constant = C.momentum_as_time_constant_schedule(1100)
lr_per_sample = learning_parameter_schedule(0.1, minibatch_size=1)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant,
unit_gain_value)
C.momentum_sgd(res.parameters,
lr_per_sample,
momentum_time_constant,
unit_gain=unit_gain_value)
C.set_default_unit_gain_value(False)
unit_gain_value = C.default_unit_gain_value()
assert not unit_gain_value
C.set_default_unit_gain_value(True)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
lr_per_sample = learning_rate_schedule([(3, 0.1), (2, 0.2), (1, 0.3)],
unit=UnitType.sample)
C.fsadagrad(res.parameters,
lr=lr_per_sample,
momentum=momentum_time_constant)
C.fsadagrad(res.parameters, lr_per_sample, momentum_time_constant,
unit_gain_value)
C.fsadagrad(res.parameters,
lr=lr_per_sample,
momentum=momentum_time_constant,
unit_gain=unit_gain_value)
gamma, inc, dec, max, min = [0.5, 1.2, 0.7, 10, 1e-8]
lr_per_sample = learning_rate_schedule([0.1, 0.2],
unit=UnitType.sample,
epoch_size=100)
C.rmsprop(res.parameters, lr_per_sample, gamma, inc, dec, max, min, True)
C.adadelta(res.parameters, lr_per_sample, use_mean_gradient=True)
def test_learner_init_legacy():
i = C.input_variable(shape=(1,), needs_gradient=True, name='a')
w = parameter(shape=(1,))
res = i * w
# for backcompatibility test
# this will be deprecated in future version
learner = sgd(res.parameters, lr=learning_rate_schedule(0.1, UnitType.sample))
assert learner._learning_rate_schedule.minibatch_size == 1 # the deprecated per sample schedule should not use compatible mode
assert learner.learning_rate() == 0.1
# for backcompatibility test
# this will be deprecated in future version
# The UnitType will provide per minibatch instruction for the learner
# this will be deprecated in future version
learner = sgd(res.parameters, lr=learning_rate_schedule(0.1, UnitType.minibatch))
assert learner.is_compatible_mode() == False
assert learner.learning_rate() == 0.1
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == 0
# for backcompatibility test, in reset learning rate, the learner won't receive the reference minibatch size from the schedule
# user will need to specify the reference minibatch size explicitly
# this will be deprecated in future version
learner = sgd(res.parameters, lr=0.1)
learner.reset_learning_rate(learning_rate_schedule([1, 2, 3], UnitType.minibatch))
assert learner.learning_rate() == 1.0
learner.minibatch_size = C.learners.IGNORE # reset to be per minibatch
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.is_compatible_mode() == True
# for backcompatibility test
# this will be deprecated in future version
learner = sgd(res.parameters, lr=learning_rate_schedule(0.1, UnitType.sample), minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.learning_rate() == 0.1
assert learner.minibatch_size == C.learners.IGNORE # the learner's reference minibatch size is still 0
# this will be deprecated in future version: This is logical invalid combination but it was the only way to use mean gradient and set learning rate in the past.
learner = sgd(res.parameters, lr=learning_rate_schedule(0.1, UnitType.sample), use_mean_gradient=True)
assert learner.is_compatible_mode() == True
assert learner.learning_rate() == 0.1
#test the override in the new version
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.minibatch_size == C.learners.IGNORE # the learner's reference minibatch size is still 0
# for backcompatibility test
# this will be deprecated in future version
# The UnitType will provide per minibatch instruction for the learner
# this will be deprecated in future version
learner = sgd(res.parameters, lr=learning_rate_schedule(0.1, UnitType.minibatch), minibatch_size=C.learners.IGNORE)
assert learner.is_compatible_mode() == True
assert learner.learning_rate() == 0.1
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
# for backcompatibility test, in reset learning rate, the learner won't receive the reference minibatch size from the schedule
# user will need to specify the reference minibatch size explicitly
# this will be deprecated in future version
learner = sgd(res.parameters, lr=0.1)
learner.reset_learning_rate(learning_rate_schedule([1, 2, 3], UnitType.minibatch))
assert learner.learning_rate() == 1.0
learner.minibatch_size = C.learners.IGNORE # reset to be per minibatch
assert learner.minibatch_size == C.learners.IGNORE
assert learner._learning_rate_schedule.minibatch_size == C.learners.IGNORE
assert learner.is_compatible_mode() == True
learner_parameter = learner.parameters
from cntk.variables import Parameter
param = learner_parameter[0]
assert isinstance(param, Parameter)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
# back compatible API test
momentum_time_constant = C.momentum_as_time_constant_schedule(1100)
lr_per_sample = learning_parameter_schedule(0.1, minibatch_size=1)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant, unit_gain_value)
C.momentum_sgd(res.parameters, lr_per_sample, momentum_time_constant, unit_gain=unit_gain_value)
C.set_default_unit_gain_value(False)
unit_gain_value = C.default_unit_gain_value()
assert not unit_gain_value
C.set_default_unit_gain_value(True)
unit_gain_value = C.default_unit_gain_value()
assert unit_gain_value
lr_per_sample = learning_rate_schedule([(3, 0.1), (2, 0.2), (1, 0.3)], unit=UnitType.sample)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum_time_constant)
C.fsadagrad(res.parameters, lr_per_sample, momentum_time_constant, unit_gain_value)
C.fsadagrad(res.parameters, lr=lr_per_sample, momentum=momentum_time_constant, unit_gain=unit_gain_value)
gamma, inc, dec, max, min = [0.5, 1.2, 0.7, 10, 1e-8]
lr_per_sample = learning_rate_schedule([0.1, 0.2], unit=UnitType.sample, epoch_size=100)
C.rmsprop(res.parameters, lr_per_sample, gamma, inc, dec, max, min, True)
C.adadelta(res.parameters, lr_per_sample, use_mean_gradient=True)
