def optimize(self, layer_list, epoch):
"""
Apply the learning rule to all the layers and update the states.
Arguments:
layer_list (list): a list of Layer objects to optimize.
epoch (int): the current epoch, needed for the Schedule object.
"""
lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
param_list = get_param_list(layer_list)
scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)
for (param, grad), states in param_list:
param.rounding = self.stochastic_round
if len(states) == 0:
states.append(self.be.zeros_like(grad))
states.append(self.be.zeros_like(grad))
grad = grad / self.be.bsz
grad = self.clip_gradient_value(grad, self.gradient_clip_value)
velocity = states[0]
velocity_backup = states[-1]
velocity_backup[:] = velocity
velocity[:] = (self.momentum_coef * velocity -
lrate * (scale_factor * grad + self.wdecay * param))
param[:] = (param + velocity * (1 + self.momentum_coef) -
self.momentum_coef * velocity_backup)
def optimize(self, layer_list, epoch):
lrate, epsilon, decay = (self.learning_rate, self.epsilon, self.decay_rate)
param_list = get_param_list(layer_list)
scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)
for (param, grad), states in param_list:
param.rounding = self.stochastic_round
if len(states) == 0:
states.append(self.be.zeros_like(grad))
states.append(self.be.zeros_like(grad))
grad = grad / self.be.bsz
grad = self.clip_gradient_value(grad, self.gradient_clip_value)
# update state
state = states[0]
state[:] = decay * state + self.be.square(grad) * (1.0 - decay)
# update velocity
velocity = states[1]
temp_velocity = - (scale_factor * grad * lrate) / self.be.sqrt(state + epsilon)
velocity[:] = self.momentum * velocity + temp_velocity
param[:] = param + self.momentum * velocity + temp_velocity
def clip_param_in_layers(layer_list, abs_bound=None):
"""
Element-wise clip all parameter tensors to between
``-abs_bound`` and ``+abs_bound`` in a list of layers.
Arguments:
layer_list (list): List of layers
be (Backend object): Backend in which the tensor resides
abs_bound (float, optional): Value to element-wise clip gradients
or parameters. Defaults to None.
"""
param_list = get_param_list(layer_list)
for (param, grad), states in param_list:
if abs_bound:
param[:] = param.backend.clip(param, -abs(abs_bound), abs(abs_bound))
def clip_param_in_layers(layer_list, abs_bound=None):
"""
Element-wise clip all parameter tensors to between
``-abs_bound`` and ``+abs_bound`` in a list of layers.
Arguments:
layer_list (list): List of layers
be (Backend object): Backend in which the tensor resides
abs_bound (float, optional): Value to element-wise clip gradients
or parameters. Defaults to None.
"""
param_list = get_param_list(layer_list)
for (param, grad), states in param_list:
if abs_bound:
param[:] = param.backend.clip(param, -abs(abs_bound), abs(abs_bound))
def optimize(self, layer_list, epoch):
self.optimizer.optimize(layer_list, epoch=epoch)
param_list = get_param_list(layer_list)
for idx, ((param, grad), states) in enumerate(param_list):
if layer_list[idx].name in ['BiasLayer']:
continue
if layer_list[idx].name in ['ConvolutionLayer']:
max_norm = self.max_kern_norm
axes = 0
else:
max_norm = self.max_col_norm
axes = 1
norm = self.be.empty_like(param)
norm[:] = self.be.sqrt(self.be.sum(self.be.square(param), axis=axes))
target_norm = self.be.clip(norm, 0., max_norm)
param[:] = param * target_norm / (norm + 1e-7)
def optimize(self, layer_list, epoch):
"""
Apply the learning rule to all the layers and update the states.
Arguments:
layer_list (list): a list of Layer objects to optimize.
epoch (int): the current epoch, needed for the Schedule object.
"""
param_list = get_param_list(layer_list)
scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)
lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
for i, params in enumerate(param_list):
if layer_list[i].use_fast_weights:
fast_params = get_fast_params(layer_list[i])
self._update_params_fast(params, fast_params, lrate, scale_factor)
else:
self._update_params(params, lrate, scale_factor)
def params(layers):
for (param, grad), states in get_param_list(layers):
yield param