def testApplyFeatureIdDropoutSkip(self):
channel = spec_pb2.FixedFeatureChannel()
text_format.Parse(
"""
vocabulary_size: 2
dropout_id: 2
dropout_keep_probability: [0.0, 1.0]
""", channel)
with tf.Graph().as_default(), self.test_session():
with tf.variable_scope('test_scope'):
ids = tf.constant([0, 1], dtype=tf.int64)
weights = tf.constant([1, 1], dtype=tf.float32)
tensors = network_units.apply_feature_id_dropout(
ids, weights, channel)
perturbed_ids, perturbed_weights = tensors[0].eval(
), tensors[1].eval()
tf.logging.info('perturbed_ids = %s', perturbed_ids)
tf.logging.info('perturbed_weights = %s', perturbed_weights)
# Given the dropout_keep_probability values specified above:
# * ID 0 is never kept, its weight is set to 0.
# * IDs 1 are always kept.
# To avoid non-determinism, we only check for specific feature IDs at
# the extremes (never/always kept).
self.assertEqual(perturbed_ids[0], channel.dropout_id)
self.assertEqual(perturbed_weights[0], 0)
self.assertEqual(perturbed_ids[1], 1)
self.assertEqual(perturbed_weights[1], 1)
def testApplyFeatureIdDropout(self):
channel = spec_pb2.FixedFeatureChannel()
text_format.Parse(
"""
vocabulary_size: 10
dropout_id: 8
dropout_keep_probability: [0.0, 0.25, 0.5, 0.75, 1.0]
""", channel)
with tf.Graph().as_default(), self.test_session():
with tf.variable_scope('test_scope'):
ids = tf.constant([0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
dtype=tf.int64)
weights = tf.constant([1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=tf.float32)
tensors = network_units.apply_feature_id_dropout(
ids, weights, channel)
perturbed_ids = tensors[0].eval()
tf.logging.info('perturbed_ids = %s', perturbed_ids)
# Given the dropout_keep_probability values specified above:
# * ID 0 is never kept.
# * IDs 1-3 are randomly kept with varying probability.
# * IDs 4-9 are always kept.
# To avoid non-determinism, we only check for specific feature IDs at
# the extremes (never/always kept). Behavior in between the extremes
# should interpolate between the two extremes.
self.assertEqual(perturbed_ids[0], channel.dropout_id)
self.assertTrue(perturbed_ids[1] in (1, channel.dropout_id))
self.assertTrue(perturbed_ids[2] in (2, channel.dropout_id))
self.assertTrue(perturbed_ids[3] in (3, channel.dropout_id))
self.assertAllEqual(perturbed_ids[4:], [4, 5, 6, 7, 8, 9])
def testApplyFeatureIdDropoutSkip(self):
channel = spec_pb2.FixedFeatureChannel()
text_format.Parse("""
vocabulary_size: 2
dropout_id: 2
dropout_keep_probability: [0.0, 1.0]
""", channel)
with tf.Graph().as_default(), self.test_session():
with tf.variable_scope('test_scope'):
ids = tf.constant([0, 1], dtype=tf.int64)
weights = tf.constant([1, 1], dtype=tf.float32)
tensors = network_units.apply_feature_id_dropout(ids, weights, channel)
perturbed_ids, perturbed_weights = tensors[0].eval(), tensors[1].eval()
tf.logging.info('perturbed_ids = %s', perturbed_ids)
tf.logging.info('perturbed_weights = %s', perturbed_weights)
# Given the dropout_keep_probability values specified above:
# * ID 0 is never kept, its weight is set to 0.
# * IDs 1 are always kept.
# To avoid non-determinism, we only check for specific feature IDs at
# the extremes (never/always kept).
self.assertEqual(perturbed_ids[0], channel.dropout_id)
self.assertEqual(perturbed_weights[0], 0)
self.assertEqual(perturbed_ids[1], 1)
self.assertEqual(perturbed_weights[1], 1)
def testApplyFeatureIdDropout(self):
channel = spec_pb2.FixedFeatureChannel()
text_format.Parse("""
vocabulary_size: 10
dropout_id: 8
dropout_keep_probability: [0.0, 0.25, 0.5, 0.75, 1.0]
""", channel)
with tf.Graph().as_default(), self.test_session():
with tf.variable_scope('test_scope'):
ids = tf.constant([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=tf.int64)
weights = tf.constant([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=tf.float32)
tensors = network_units.apply_feature_id_dropout(ids, weights, channel)
perturbed_ids = tensors[0].eval()
tf.logging.info('perturbed_ids = %s', perturbed_ids)
# Given the dropout_keep_probability values specified above:
# * ID 0 is never kept.
# * IDs 1-3 are randomly kept with varying probability.
# * IDs 4-9 are always kept.
# To avoid non-determinism, we only check for specific feature IDs at
# the extremes (never/always kept). Behavior in between the extremes
# should interpolate between the two extremes.
self.assertEqual(perturbed_ids[0], channel.dropout_id)
self.assertTrue(perturbed_ids[1] in (1, channel.dropout_id))
self.assertTrue(perturbed_ids[2] in (2, channel.dropout_id))
self.assertTrue(perturbed_ids[3] in (3, channel.dropout_id))
self.assertAllEqual(perturbed_ids[4:], [4, 5, 6, 7, 8, 9])
def fetch_differentiable_fixed_embeddings(comp, state, stride,
during_training):
"""Looks up fixed features with separate, differentiable, embedding lookup.
Args:
comp: Component whose fixed features we wish to look up.
state: live MasterState object for the component.
stride: Tensor containing current batch * beam size.
during_training: True if this is being called from a training code path.
This controls, e.g., the use of feature ID dropout.
Returns:
state handle: updated state handle to be used after this call
fixed_embeddings: list of NamedTensor objects
"""
_validate_embedded_fixed_features(comp)
num_channels = len(comp.spec.fixed_feature)
if not num_channels:
return state.handle, []
state.handle, indices, ids, weights, num_steps = (
dragnn_ops.bulk_fixed_features(state.handle,
component=comp.name,
num_channels=num_channels))
fixed_embeddings = []
for channel, feature_spec in enumerate(comp.spec.fixed_feature):
differentiable_or_constant = ('constant' if feature_spec.is_constant
else 'differentiable')
tf.logging.info('[%s] Adding %s fixed feature "%s"', comp.name,
differentiable_or_constant, feature_spec.name)
if during_training and feature_spec.dropout_id >= 0:
ids[channel], weights[
channel] = network_units.apply_feature_id_dropout(
ids[channel], weights[channel], feature_spec)
size = stride * num_steps * feature_spec.size
fixed_embedding = network_units.embedding_lookup(
comp.get_variable(network_units.fixed_embeddings_name(channel)),
indices[channel], ids[channel], weights[channel], size)
if feature_spec.is_constant:
fixed_embedding = tf.stop_gradient(fixed_embedding)
fixed_embeddings.append(
network_units.NamedTensor(fixed_embedding, feature_spec.name))
return state.handle, fixed_embeddings
def fetch_differentiable_fixed_embeddings(comp, state, stride, during_training):
"""Looks up fixed features with separate, differentiable, embedding lookup.
Args:
comp: Component whose fixed features we wish to look up.
state: live MasterState object for the component.
stride: Tensor containing current batch * beam size.
during_training: True if this is being called from a training code path.
This controls, e.g., the use of feature ID dropout.
Returns:
state handle: updated state handle to be used after this call
fixed_embeddings: list of NamedTensor objects
"""
_validate_embedded_fixed_features(comp)
num_channels = len(comp.spec.fixed_feature)
if not num_channels:
return state.handle, []
state.handle, indices, ids, weights, num_steps = (
dragnn_ops.bulk_fixed_features(
state.handle, component=comp.name, num_channels=num_channels))
fixed_embeddings = []
for channel, feature_spec in enumerate(comp.spec.fixed_feature):
differentiable_or_constant = ('constant' if feature_spec.is_constant else
'differentiable')
tf.logging.info('[%s] Adding %s fixed feature "%s"', comp.name,
differentiable_or_constant, feature_spec.name)
if during_training and feature_spec.dropout_id >= 0:
ids[channel], weights[channel] = network_units.apply_feature_id_dropout(
ids[channel], weights[channel], feature_spec)
size = stride * num_steps * feature_spec.size
fixed_embedding = network_units.embedding_lookup(
comp.get_variable(network_units.fixed_embeddings_name(channel)),
indices[channel], ids[channel], weights[channel], size)
if feature_spec.is_constant:
fixed_embedding = tf.stop_gradient(fixed_embedding)
fixed_embeddings.append(
network_units.NamedTensor(fixed_embedding, feature_spec.name))
return state.handle, fixed_embeddings