def test_with_get_output_kw(self, objective, get_output):
loss_function, target = Mock(), Mock()
loss_function.return_value = np.array([1, 2, 3])
objective(
[1, 2, 3], loss_function=loss_function, target=target,
get_output_kw={'i_was': 'here'},
)
get_output.assert_called_with(3, deterministic=False, i_was='here')
def test_with_get_output_kw(self, objective, get_output):
loss_function, target = Mock(), Mock()
loss_function.return_value = np.array([1, 2, 3])
objective(
[1, 2, 3], loss_function=loss_function, target=target,
get_output_kw={'i_was': 'here'},
)
get_output.assert_called_with(3, deterministic=False, i_was='here')
def test_with_defaults(self, objective, get_output):
loss_function, target = Mock(), Mock()
loss_function.return_value = np.array([1, 2, 3])
result = objective([1, 2, 3], loss_function=loss_function, target=target)
assert result == 2.0
get_output.assert_called_with(3, deterministic=False)
loss_function.assert_called_with(get_output.return_value, target)
def test_with_defaults(self, objective, get_output):
loss_function, target = Mock(), Mock()
loss_function.return_value = np.array([1, 2, 3])
result = objective(
[1, 2, 3], loss_function=loss_function, target=target)
assert result == 2.0
get_output.assert_called_with(3, deterministic=False)
loss_function.assert_called_with(get_output.return_value, target)
def regularization_objective(layers, lambda1=0., lambda2=0., *args, **kwargs):
# default loss
losses = objective(layers, *args, **kwargs)
# get the layers' weights, but only those that should be regularized
# (i.e. not the biases)
weights = get_all_params(layers[-1], regularizable=True)
# sum of absolute weights for L1
sum_abs_weights = sum([abs(w).sum() for w in weights])
# sum of squared weights for L2
sum_squared_weights = sum([(w ** 2).sum() for w in weights])
# add weights to regular loss
losses += lambda1 * sum_abs_weights + lambda2 * sum_squared_weights
return losses
def regularization_objective(layers, lambda1=0., lambda2=0., *args, **kwargs):
# default loss
losses = objective(layers, *args, **kwargs)
# get the layers' weights, but only those that should be regularized
# (i.e. not the biases)
weights = get_all_params(layers[-1], regularizable=True)
# sum of absolute weights for L1
sum_abs_weights = sum([abs(w).sum() for w in weights])
# sum of squared weights for L2
sum_squared_weights = sum([(w**2).sum() for w in weights])
# add weights to regular loss
losses += lambda1 * sum_abs_weights + lambda2 * sum_squared_weights
return losses
def regularization_objective(layers, lambda1=0., lambda2=0., *args, **kwargs):
# default loss
losses = objective(layers, *args, **kwargs)
# get layer weights except for the biases
weights = get_all_params(layers[-1], regularizable=True)
regularization_term = 0.0
# sum of abs weights for L1 regularization
if lambda1 != 0.0:
sum_abs_weights = sum([abs(w).sum() for w in weights])
regularization_term += (lambda1 * sum_abs_weights)
# sum of squares (sum(theta^2))
if lambda2 != 0.0:
sum_squared_weights = (1 / 2.0) * sum([(w ** 2).sum() for w in weights])
regularization_term += (lambda2 * sum_squared_weights)
# add weights to regular loss
losses += regularization_term
return losses
def _create_iter_funcs(self, layers, objective, update, output_type):
y_batch = output_type('y_batch')
objective_kw = self._get_params_for('objective')
loss_train = objective(layers, target=y_batch, **objective_kw)
loss_eval = objective(layers,
target=y_batch,
deterministic=True,
**objective_kw)
output_layer = self._output_layers
predict_proba = get_output(output_layer, None, deterministic=True)
if not self.regression:
predict = predict_proba[0].argmax(axis=1)
accuracy = T.mean(T.eq(predict, y_batch))
else:
accuracy = loss_eval
scores_train = [
s[1](predict_proba, y_batch) for s in self.scores_train
]
scores_valid = [
s[1](predict_proba, y_batch) for s in self.scores_valid
]
all_params = self.get_all_params(trainable=True)
grads = T.grad(loss_train, all_params)
for idx, param in enumerate(all_params):
grad_scale = getattr(param.tag, 'grad_scale', 1)
if grad_scale != 1:
grads[idx] *= grad_scale
update_params = self._get_params_for('update')
updates = update(grads, all_params, **update_params)
input_layers = [
layer for layer in layers.values()
if isinstance(layer, InputLayer)
]
X_inputs = [
theano.In(input_layer.input_var, name=input_layer.name)
for input_layer in input_layers
]
inputs = X_inputs + [theano.In(y_batch, name="y")]
train_iter = theano.function(
inputs=inputs,
outputs=[loss_train] + scores_train,
updates=updates,
allow_input_downcast=True,
)
eval_iter = theano.function(
inputs=inputs,
outputs=[loss_eval, accuracy] + scores_valid,
allow_input_downcast=True,
)
predict_iter = theano.function(
inputs=X_inputs,
outputs=predict_proba,
allow_input_downcast=True,
)
return train_iter, eval_iter, predict_iter
