def encode(self, input_dict):
"""Wrapper around :meth:`self._encode() <_encode>` method.
Here name, initializer and dtype are set in the variable scope and then
:meth:`self._encode() <_encode>` method is called.
Args:
input_dict (dict): see :meth:`self._encode() <_encode>` docs.
Returns:
see :meth:`self._encode() <_encode>` docs.
"""
if not self._compiled:
if 'regularizer' not in self._params:
if self._model and 'regularizer' in self._model.params:
self._params['regularizer'] = copy.deepcopy(
self._model.params['regularizer'])
self._params['regularizer_params'] = copy.deepcopy(
self._model.params['regularizer_params'])
if 'regularizer' in self._params:
init_dict = self._params.get('regularizer_params', {})
self._params['regularizer'] = self._params['regularizer'](
**init_dict)
if self._params['dtype'] == 'mixed':
self._params['regularizer'] = mp_regularizer_wrapper(
self._params['regularizer'], )
if self._params['dtype'] == 'mixed':
self._params['dtype'] = tf.float16
self._compiled = True
with tf.variable_scope(self._name, dtype=self.params['dtype']):
return self._encode(self._cast_types(input_dict))
def __init__(self, params, model, name="encoder", mode='train'):
"""Encoder constructor.
Note that encoder constructors should not modify TensorFlow graph, all
graph construction should happen in the :meth:`self._encode() <_encode>`
method.
Args:
params (dict): parameters describing the encoder.
All supported parameters are listed in :meth:`get_required_params`,
:meth:`get_optional_params` functions.
model (instance of a class derived from :class:`Model<models.model.Model>`):
parent model that created this encoder.
Could be None if no model access is required for the use case.
name (str): name for encoder variable scope.
mode (str): mode encoder is going to be run in.
Could be "train", "eval" or "infer".
Config parameters:
* **initializer** --- any valid TensorFlow initializer. If no initializer
is provided, model initializer will be used.
* **initializer_params** (dict) --- dictionary that will be passed to
initializer ``__init__`` method.
* **regularizer** --- and valid TensorFlow regularizer. If no regularizer
is provided, model regularizer will be used.
* **regularizer_params** (dict) --- dictionary that will be passed to
regularizer ``__init__`` method.
* **dtype** --- model dtype. Could be either ``tf.float16``, ``tf.float32``
or "mixed". For details see
:ref:`mixed precision training <mixed_precision>` section in docs. If no
dtype is provided, model dtype will be used.
"""
check_params(params, self.get_required_params(), self.get_optional_params())
self._params = copy.deepcopy(params)
self._model = model
if 'dtype' not in self._params:
if self._model:
self._params['dtype'] = self._model.params['dtype']
else:
self._params['dtype'] = tf.float32
if 'regularizer' not in self._params:
if self._model and 'regularizer' in self._model.params:
self._params['regularizer'] = self._model.params['regularizer']
self._params['regularizer_params'] = self._model.params['regularizer_params']
if 'regularizer' in self._params:
init_dict = self._params.get('regularizer_params', {})
self._params['regularizer'] = self._params['regularizer'](**init_dict)
if self._params['dtype'] == 'mixed':
self._params['regularizer'] = mp_regularizer_wrapper(
self._params['regularizer'],
)
if self._params['dtype'] == 'mixed':
self._params['dtype'] = tf.float16
self._name = name
self._mode = mode
def __init__(self, params, model, name="decoder", mode='train'):
"""Decoder constructor.
Note that decoder constructors should not modify TensorFlow graph, all
graph construction should happen in the :meth:`self._decode() <_decode>`
method.
Args:
params (dict): parameters describing the decoder.
All supported parameters are listed in :meth:`get_required_params`,
:meth:`get_optional_params` functions.
model (instance of a class derived from :class:`Model<models.model.Model>`):
parent model that created this decoder.
Could be None if no model access is required for the use case.
name (str): name for decoder variable scope.
mode (str): mode decoder is going to be run in.
Could be "train", "eval" or "infer".
Config parameters:
* **initializer** --- any valid TensorFlow initializer. If no initializer
is provided, model initializer will be used.
* **initializer_params** (dict) --- dictionary that will be passed to
initializer ``__init__`` method.
* **regularizer** --- and valid TensorFlow regularizer. If no regularizer
is provided, model regularizer will be used.
* **regularizer_params** (dict) --- dictionary that will be passed to
regularizer ``__init__`` method.
* **dtype** --- model dtype. Could be either ``tf.float16``, ``tf.float32``
or "mixed". For details see
:ref:`mixed precision training <mixed_precision>` section in docs. If no
dtype is provided, model dtype will be used.
"""
check_params(params, self.get_required_params(), self.get_optional_params())
self._params = copy.deepcopy(params)
self._model = model
if 'dtype' not in self._params:
if self._model:
self._params['dtype'] = self._model.params['dtype']
else:
self._params['dtype'] = tf.float32
if 'regularizer' not in self._params:
if self._model and 'regularizer' in self._model.params:
self._params['regularizer'] = self._model.params['regularizer']
self._params['regularizer_params'] = self._model.params['regularizer_params']
if 'regularizer' in self._params:
init_dict = self._params.get('regularizer_params', {})
self._params['regularizer'] = self._params['regularizer'](**init_dict)
if self._params['dtype'] == 'mixed':
self._params['regularizer'] = mp_regularizer_wrapper(
self._params['regularizer'],
)
if self._params['dtype'] == 'mixed':
self._params['dtype'] = tf.float16
self._name = name
self._mode = mode
def test_regularization_mixed(self):
n_samples = 3
n_hid = 2
scale_init = 1e-4
wd = 1e-4
X = np.ones((n_samples, n_hid)) / n_hid
y = np.ones((n_samples, 1)) * scale_init
dtype = tf.float16
regularizer = mp_regularizer_wrapper(tf.contrib.layers.l2_regularizer(wd))
with tf.Graph().as_default() as g:
x_ph = tf.placeholder(dtype, [n_samples, n_hid])
y_ph = tf.placeholder(dtype, [n_samples, 1])
y_pred = tf.layers.dense(
x_ph, 1, kernel_regularizer=regularizer,
use_bias=False,
kernel_initializer=tf.constant_initializer(scale_init, dtype=dtype),
)
loss = tf.reduce_mean((y_ph - y_pred) ** 2)
reg_loss = tf.losses.get_regularization_loss()
loss += tf.cast(reg_loss, loss.dtype)
opt = MixedPrecisionOptimizerWrapper(tf.train.AdamOptimizer())
grad = opt.compute_gradients(loss)[0][0]
with self.test_session(g, use_gpu=True) as sess:
sess.run(tf.global_variables_initializer())
reg_loss_val, grad_val = sess.run([reg_loss, grad],
{x_ph: X, y_ph: y})
self.assertAlmostEqual(reg_loss_val, 0.0)
self.assertEqual(reg_loss.name, "Const_1:0")
npt.assert_allclose(grad_val, np.ones((2, 1)) * 1e-8, atol=1e-11)
def test_regularization_mixed(self):
n_samples = 3
n_hid = 2
scale_init = 1e-4
wd = 1e-4
X = np.ones((n_samples, n_hid)) / n_hid
y = np.ones((n_samples, 1)) * scale_init
dtype = tf.float16
regularizer = mp_regularizer_wrapper(tf.contrib.layers.l2_regularizer(wd))
with tf.Graph().as_default() as g:
x_ph = tf.placeholder(dtype, [n_samples, n_hid])
y_ph = tf.placeholder(dtype, [n_samples, 1])
y_pred = tf.layers.dense(
x_ph, 1, kernel_regularizer=regularizer,
use_bias=False,
kernel_initializer=tf.constant_initializer(scale_init, dtype=dtype),
)
loss = tf.reduce_mean((y_ph - y_pred) ** 2)
reg_loss = tf.losses.get_regularization_loss()
loss += tf.cast(reg_loss, loss.dtype)
opt = MixedPrecisionOptimizerWrapper(tf.train.AdamOptimizer())
grad = opt.compute_gradients(loss)[0][0]
with self.test_session(g, use_gpu=True) as sess:
sess.run(tf.global_variables_initializer())
reg_loss_val, grad_val = sess.run([reg_loss, grad],
{x_ph: X, y_ph: y})
self.assertAlmostEqual(reg_loss_val, 0.0)
self.assertEqual(reg_loss.name, "Const_1:0")
npt.assert_allclose(grad_val, np.ones((2, 1)) * 1e-8, atol=1e-11)
