def testEmbedding(self):
x = np.random.random_integers(1, high=8, size=(3, 5))
with self.test_session() as session:
y = common_layers.embedding(x, 10, 16)
session.run(tf.global_variables_initializer())
res = session.run(y)
self.assertEqual(res.shape, (3, 5, 16))
def testEmbedding(self):
x = np.random.random_integers(1, high=8, size=(3, 5))
with self.test_session() as session:
y = common_layers.embedding(x, 10, 16)
session.run(tf.global_variables_initializer())
res = session.run(y)
self.assertEqual(res.shape, (3, 5, 16))
def testFlatten4D3D(self):
x = np.random.random_integers(1, high=8, size=(3, 5, 2))
with self.test_session() as session:
y = common_layers.flatten4d3d(common_layers.embedding(x, 10, 7))
session.run(tf.global_variables_initializer())
res = session.run(y)
self.assertEqual(res.shape, (3, 5 * 2, 7))
def transformer_prepare_encoder(inputs, target_space, hparams):
"""Prepare one shard of the model for the encoder.
Args:
inputs: a Tensor.
target_space: a Tensor.
hparams: run hyperparameters
Returns:
encoder_input: a Tensor, bottom of encoder stack
encoder_self_attention_bias: a Tensor, containing large negative values
to implement masked attention and possibly baises for diagonal
alignments
encoder_padding: a Tensor
"""
# Flatten inputs.
ishape_static = inputs.shape.as_list()
encoder_input = inputs
encoder_padding = common_attention.embedding_to_padding(encoder_input)
encoder_self_attention_bias = common_attention.attention_bias_ignore_padding(
encoder_padding)
# Append target_space_id embedding to inputs.
emb_target_space = common_layers.embedding(
target_space, 32, ishape_static[-1], name="target_space_embedding")
emb_target_space = tf.reshape(emb_target_space, [1, 1, -1])
encoder_input += emb_target_space
if hparams.pos == "timing":
encoder_input = common_attention.add_timing_signal_1d(encoder_input)
return (encoder_input, encoder_self_attention_bias, encoder_padding)
def testFlatten4D3D(self):
x = np.random.random_integers(1, high=8, size=(3, 5, 2))
with self.test_session() as session:
y = common_layers.flatten4d3d(common_layers.embedding(x, 10, 7))
session.run(tf.global_variables_initializer())
res = session.run(y)
self.assertEqual(res.shape, (3, 5 * 2, 7))
def transformer_prepare_encoder(inputs, target_space, hparams):
"""Prepare one shard of the model for the encoder.
Args:
inputs: a Tensor.
target_space: a Tensor.
hparams: run hyperparameters
Returns:
encoder_input: a Tensor, bottom of encoder stack
encoder_self_attention_bias: a Tensor, containing large negative values
to implement masked attention and possibly baises for diagonal
alignments
encoder_padding: a Tensor
"""
# Flatten inputs.
ishape_static = inputs.shape.as_list()
encoder_input = inputs
encoder_padding = common_attention.embedding_to_padding(encoder_input)
encoder_self_attention_bias = common_attention.attention_bias_ignore_padding(
encoder_padding)
# Append target_space_id embedding to inputs.
emb_target_space = common_layers.embedding(target_space,
32,
ishape_static[-1],
name="target_space_embedding")
emb_target_space = tf.reshape(emb_target_space, [1, 1, -1])
encoder_input += emb_target_space
if hparams.pos == "timing":
encoder_input = common_attention.add_timing_signal_1d(encoder_input)
return (encoder_input, encoder_self_attention_bias, encoder_padding)
def inputs_bottom_simple(self, x):
with tf.variable_scope(self.name):
return common_layers.embedding(
x,
self._vocab_size,
self._body_input_depth,
multiplier=self._body_input_depth**0.5 if
self._model_hparams.multiply_embedding_mode == "sqrt_depth" else 1.0)
def bottom(self, x):
with tf.variable_scope(self.name):
return common_layers.embedding(
x,
self._vocab_size,
self._body_input_depth,
multiplier=self._body_input_depth**0.5 if
self._model_hparams.multiply_embedding_mode == "sqrt_depth" else 1.0)
def targets_bottom_simple(self, inputs):
with tf.variable_scope(self.name):
# Reshape inputs to 2-d tensor and embed the RGB pixel values.
inputs = common_layers.flatten4d3d(inputs)
ret = common_layers.embedding(inputs, self.targets_dimensionality,
self._body_input_depth,
name="input_rgb_embedding")
if self._model_hparams.multiply_embedding_mode == "sqrt_depth":
ret *= self._body_input_depth**0.5
return ret
def targets_bottom(self, inputs):
with tf.variable_scope(self.name):
# Reshape inputs to 2-d tensor and embed the RGB pixel values.
inputs = common_layers.flatten4d3d(inputs)
ret = common_layers.embedding(
inputs,
self.top_dimensionality,
self._body_input_depth,
name="input_rgb_embedding")
if self._model_hparams.multiply_embedding_mode == "sqrt_depth":
ret *= self._body_input_depth**0.5
return ret
def embed_target_space(target_space_id, hidden_size):
target_space_emb = common_layers.embedding(target_space_id,
32,
hidden_size,
name="target_space_embedding")
return tf.reshape(target_space_emb, [1, 1, 1, -1])
def embed_target_space(target_space_id, hidden_size):
target_space_emb = common_layers.embedding(
target_space_id, 32, hidden_size, name="target_space_embedding")
return tf.reshape(target_space_emb, [1, 1, 1, -1])
