def test_regularization_mixed(self):
n_samples = 3
n_hid = 2
scale_init = 1e-4
wd = 1e-4
X = np.ones((n_samples, n_hid)) / n_hid
y = np.ones((n_samples, 1)) * scale_init
dtype = tf.float16
# pylint: disable=no-member
regularizer = mp_regularizer_wrapper(tf.contrib.layers.l2_regularizer(wd))
with tf.Graph().as_default() as g:
x_ph = tf.placeholder(dtype, [n_samples, n_hid])
y_ph = tf.placeholder(dtype, [n_samples, 1])
y_pred = tf.layers.dense(
x_ph, 1, kernel_regularizer=regularizer,
use_bias=False,
kernel_initializer=tf.constant_initializer(scale_init, dtype=dtype),
)
loss = tf.reduce_mean((y_ph - y_pred) ** 2)
reg_loss = tf.losses.get_regularization_loss()
loss += tf.cast(reg_loss, loss.dtype)
opt = MixedPrecisionOptimizerWrapper(tf.train.AdamOptimizer())
grad = opt.compute_gradients(loss)[0][0]
with self.test_session(g, use_gpu=True) as sess:
sess.run(tf.global_variables_initializer())
reg_loss_val, grad_val = sess.run([reg_loss, grad],
{x_ph: X, y_ph: y})
self.assertAlmostEqual(reg_loss_val, 0.0)
self.assertEqual(reg_loss.name, "Const_1:0")
npt.assert_allclose(grad_val, np.ones((2, 1)) * 1e-8, atol=1e-11)
def encode(self, input_dict):
"""Wrapper around :meth:`self._encode() <_encode>` method.
Here name, initializer and dtype are set in the variable scope and then
:meth:`self._encode() <_encode>` method is called.
Args:
input_dict (dict): see :meth:`self._encode() <_encode>` docs.
Returns:
see :meth:`self._encode() <_encode>` docs.
"""
if not self._compiled:
if 'regularizer' not in self._params:
if self._model and 'regularizer' in self._model.params:
self._params['regularizer'] = copy.deepcopy(
self._model.params['regularizer'])
self._params['regularizer_params'] = copy.deepcopy(
self._model.params['regularizer_params'])
if 'regularizer' in self._params:
init_dict = self._params.get('regularizer_params', {})
self._params['regularizer'] = self._params['regularizer'](
**init_dict)
if self._params['dtype'] == 'mixed':
self._params['regularizer'] = mp_regularizer_wrapper(
self._params['regularizer'], )
if self._params['dtype'] == 'mixed':
self._params['dtype'] = tf.float16
self._compiled = True
with tf.variable_scope(self._name, dtype=self.params['dtype']):
return self._encode(self._cast_types(input_dict))