def testImagetransformer(self, net, hparams):
batch_size = 3
size = 7
vocab_size = 256
p_hparams = problem_hparams.test_problem_hparams(vocab_size,
vocab_size,
hparams)
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, 1, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, size, size, 3))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = net(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
if hparams.likelihood == common_image_attention.DistributionType.CAT:
expected = (batch_size, size, size, 3, vocab_size)
else:
expected = (batch_size, size, size, hparams.num_mixtures * 10)
self.assertEqual(res.shape, expected)
def testVqaAttentionBaseline(self):
batch_size = 3
image_size = 448
vocab_size = 100
num_classes = 10
question_length = 5
answer_length = 10
x = 2 * np.random.rand(batch_size, image_size, image_size, 3) - 1
q = np.random.random_integers(
1, high=vocab_size - 1, size=(batch_size, question_length, 1, 1))
a = np.random.random_integers(
0, high=num_classes, size=(batch_size, answer_length, 1, 1))
hparams = vqa_attention.vqa_attention_base()
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
p_hparams.input_modality["inputs"] = (registry.Modalities.IMAGE, None)
p_hparams.input_modality["question"] = (registry.Modalities.SYMBOL,
vocab_size)
p_hparams.target_modality = (registry.Modalities.CLASS_LABEL
+ ":multi_label", num_classes + 1)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.float32),
"question": tf.constant(q, dtype=tf.int32),
"targets": tf.constant(a, dtype=tf.int32),
}
model = vqa_attention.VqaAttentionBaseline(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, losses = model(features)
session.run(tf.global_variables_initializer())
logits_, losses_ = session.run([logits, losses])
self.assertEqual(logits_.shape, (batch_size, 1, 1, 1, num_classes + 1))
self.assertEqual(losses_["training"].shape, ())
def testVqaAttentionBaseline(self):
batch_size = 3
image_size = 448
vocab_size = 100
num_classes = 10
question_length = 5
answer_length = 10
x = 2 * np.random.rand(batch_size, image_size, image_size, 3) - 1
q = np.random.randint(1,
high=vocab_size,
size=(batch_size, question_length, 1, 1))
a = np.random.randint(num_classes + 1,
size=(batch_size, answer_length, 1, 1))
hparams = vqa_attention.vqa_attention_base()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, num_classes + 1, hparams)
p_hparams.modality["inputs"] = modalities.ModalityType.IMAGE
p_hparams.modality["targets"] = modalities.ModalityType.MULTI_LABEL
p_hparams.modality["question"] = modalities.ModalityType.SYMBOL
p_hparams.vocab_size["question"] = vocab_size
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.float32),
"question": tf.constant(q, dtype=tf.int32),
"targets": tf.constant(a, dtype=tf.int32),
}
model = vqa_attention.VqaAttentionBaseline(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, losses = model(features)
session.run(tf.global_variables_initializer())
logits_, losses_ = session.run([logits, losses])
self.assertEqual(logits_.shape, (batch_size, 1, 1, 1, num_classes + 1))
self.assertEqual(losses_["training"].shape, ())
def get_model(hparams=None,
mode=tf.estimator.ModeKeys.TRAIN,
has_input=True,
model_cls=mtf_transformer.MtfTransformer):
if hparams is None:
hparams = mtf_transformer.mtf_transformer_single()
hparams.max_length = INPUT_LENGTH
hparams.batch_size = BATCH_SIZE
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE, VOCAB_SIZE,
hparams)
if not has_input:
p_hparams.input_modality = {}
hparams.problem_hparams = p_hparams
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return model_cls(hparams, mode, p_hparams), features, hparams
def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
has_input=True, model_cls=transformer.Transformer):
if hparams is None:
hparams = transformer.transformer_tiny()
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
VOCAB_SIZE,
hparams)
if not has_input:
del p_hparams.modality["inputs"]
hparams.problem_hparams = p_hparams
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return model_cls(hparams, mode, p_hparams), features
def testNeuralGPU(self):
hparams = common_hparams.basic_params1()
batch_size = 3
input_length = 5
target_length = input_length
input_vocab_size = 9
target_vocab_size = 11
p_hparams = problem_hparams.test_problem_hparams(input_vocab_size,
target_vocab_size)
inputs = -1 + np.random.random_integers(
input_vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
target_vocab_size, size=(batch_size, target_length, 1, 1))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32)
}
model = neural_gpu.NeuralGPU(hparams, tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size, target_length, 1, 1,
target_vocab_size))
def _testTransformer(self, net):
batch_size = 3
input_length = 5
target_length = 7
vocab_size = 9
hparams = transformer.transformer_tiny()
p_hparams = problem_hparams.test_problem_hparams(
hparams, vocab_size, vocab_size)
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, target_length, 1, 1))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = net(hparams, p_hparams)
shadred_logits, _, _ = model.model_fn(features, True)
logits = tf.concat(shadred_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape,
(batch_size, target_length, 1, 1, vocab_size))
def testTransformer(self):
batch_size = 3
input_length = 5
target_length = 7
vocab_size = 9
hparams = transformer_revnet_test()
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, target_length, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = transformer_revnet.TransformerRevnet(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
grads = tf.gradients(
tf.reduce_mean(logits), [features["inputs"]] + tf.global_variables())
grads = [g for g in grads if g is not None]
with self.test_session() as session:
session.run(tf.global_variables_initializer())
logits_val, _ = session.run([logits, grads])
self.assertEqual(logits_val.shape, (batch_size, target_length, 1, 1,
vocab_size))
def get_model(hparams, has_input=True):
hparams.layer_prepostprocess_dropout = 0.0
hparams.hidden_size = 16
hparams.num_heads = 1
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE, VOCAB_SIZE,
hparams)
if not has_input:
del p_hparams.modality["inputs"]
hparams.problem_hparams = p_hparams
inputs = np.random.randint(VOCAB_SIZE,
size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = np.random.randint(VOCAB_SIZE,
size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return (evolved_transformer.EvolvedTransformer(hparams,
tf.estimator.ModeKeys.TRAIN,
p_hparams), features)
def testTransformer(self):
batch_size = 3
input_length = 5
target_length = 7
vocab_size = 9
hparams = rev_transformer_test()
p_hparams = problem_hparams.test_problem_hparams(
hparams, vocab_size, vocab_size)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, target_length, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = rev_transformer.RevTransformer(hparams,
tf.contrib.learn.ModeKeys.TRAIN,
p_hparams)
sharded_logits, _ = model.model_fn(features)
logits = tf.concat(sharded_logits, 0)
grads = tf.gradients(tf.reduce_mean(logits),
[features["inputs"]] + tf.global_variables())
grads = [g for g in grads if g is not None]
with self.test_session() as session:
session.run(tf.global_variables_initializer())
logits_val, _ = session.run([logits, grads])
self.assertEqual(logits_val.shape,
(batch_size, target_length, 1, 1, vocab_size))
def testNeuralGPU(self):
hparams = common_hparams.basic_params1()
batch_size = 3
input_length = 5
target_length = input_length
input_vocab_size = 9
target_vocab_size = 11
p_hparams = problem_hparams.test_problem_hparams(
input_vocab_size, target_vocab_size, hparams)
inputs = np.random.randint(input_vocab_size,
size=(batch_size, input_length, 1, 1))
targets = np.random.randint(target_vocab_size,
size=(batch_size, target_length, 1, 1))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32)
}
model = neural_gpu.NeuralGPU(hparams, tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape,
(batch_size, target_length, 1, 1, target_vocab_size))
def getModel(self,
hparams,
mode=tf.estimator.ModeKeys.TRAIN,
has_input=True):
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(
VOCAB_SIZE, VOCAB_SIZE)
if not has_input:
p_hparams.input_modality = {}
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32, name="inputs"),
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
return transformer.Transformer(hparams, mode, p_hparams), features
def _test_xception(self, img_size):
vocab_size = 9
batch_size = 3
x = np.random.random_integers(0,
high=255,
size=(batch_size, img_size, img_size, 3))
y = np.random.random_integers(1,
high=vocab_size - 1,
size=(batch_size, 1, 1, 1))
hparams = xception.xception_tiny()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
p_hparams.input_modality["inputs"] = modalities.ImageModality(hparams)
p_hparams.target_modality = modalities.ClassLabelModality(
hparams, vocab_size)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = xception.Xception(hparams, tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size, 1, 1, 1, vocab_size))
def _test_model(self, model_cls, hparams):
"""Test a Translation Nas Net model."""
tf.reset_default_graph()
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
hparams.hidden_size = _HIDDEN_SIZE
p_hparams = problem_hparams.test_problem_hparams(
_VOCAB_SIZE, _VOCAB_SIZE, hparams)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
_VOCAB_SIZE, size=(_BATCH_SIZE, _INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
_VOCAB_SIZE, size=(_BATCH_SIZE, _TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32, name="inputs"),
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
model = model_cls(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape,
(_BATCH_SIZE, _TARGET_LENGTH, 1, 1, _VOCAB_SIZE))
def _testResnet(self, img_size, output_size):
vocab_size = 9
batch_size = 2
x = np.random.random_integers(0,
high=255,
size=(batch_size, img_size, img_size, 3))
y = np.random.random_integers(1,
high=vocab_size - 1,
size=(batch_size, 1, 1, 1))
hparams = resnet_tiny_cpu()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size)
p_hparams.input_modality["inputs"] = (registry.Modalities.IMAGE, None)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = resnet.Resnet(hparams, tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape,
(batch_size, ) + output_size + (1, vocab_size))
def testTransformerAEOnDVQ(self):
batch_size = 3
input_length = 5
target_length = 16
vocab_size = 9
hparams = transformer_vae.transformer_ae_small()
hparams.bottleneck_kind = "dvq"
hparams.dp_strength = 0
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
hparams.problem_hparams = p_hparams
inputs = np.random.randint(vocab_size,
size=(batch_size, input_length, 1, 1))
targets = np.random.randint(vocab_size,
size=(batch_size, target_length, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
tf.train.create_global_step()
model = transformer_vae.TransformerAE(hparams,
tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
logits_val = session.run(logits)
self.assertEqual(logits_val.shape,
(batch_size, target_length, 1, 1, vocab_size))
def testLSTMSeq2SeqAttention(self):
vocab_size = 9
x = np.random.random_integers(1,
high=vocab_size - 1,
size=(3, 5, 1, 1))
y = np.random.random_integers(1,
high=vocab_size - 1,
size=(3, 6, 1, 1))
hparams = lstm.lstm_attention()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size)
x = tf.constant(x, dtype=tf.int32)
x = tf.placeholder_with_default(x, shape=[None, None, 1, 1])
with self.test_session() as session:
features = {
"inputs": x,
"targets": tf.constant(y, dtype=tf.int32),
}
model = lstm.LSTMSeq2seqAttention(hparams,
tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 6, 1, 1, vocab_size))
def testTransformerAEOnDVQ(self):
batch_size = 3
input_length = 5
target_length = 16
vocab_size = 9
hparams = transformer_vae.transformer_ae_small()
hparams.bottleneck_kind = "dvq"
hparams.dp_strength = 0
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
hparams.problem_hparams = p_hparams
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, target_length, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
tf.train.create_global_step()
model = transformer_vae.TransformerAE(hparams, tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
logits_val = session.run(logits)
self.assertEqual(logits_val.shape,
(batch_size, target_length, 1, 1, vocab_size))
def testLSTMSeq2SeqAttention(self):
vocab_size = 9
x = np.random.random_integers(1,
high=vocab_size - 1,
size=(3, 5, 1, 1))
y = np.random.random_integers(1,
high=vocab_size - 1,
size=(3, 6, 1, 1))
hparams = lstm.lstm_attention()
p_hparams = problem_hparams.test_problem_hparams(
hparams, vocab_size, vocab_size)
x = tf.constant(x, dtype=tf.int32)
x._shape = tf.TensorShape([None, None, 1, 1])
with self.test_session() as session:
features = {
"inputs": x,
"targets": tf.constant(y, dtype=tf.int32),
}
model = lstm.LSTMSeq2SeqAttention(hparams,
tf.contrib.learn.ModeKeys.TRAIN,
p_hparams)
sharded_logits, _, _ = model.model_fn(features)
logits = tf.concat(sharded_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 6, 1, 1, vocab_size))
def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
has_input=True, model_cls=mtf_transformer.MtfTransformer):
if hparams is None:
hparams = mtf_transformer.mtf_transformer_single()
hparams.max_length = INPUT_LENGTH
hparams.batch_size = BATCH_SIZE
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
VOCAB_SIZE,
hparams)
if not has_input:
del p_hparams.modality["inputs"]
hparams.problem_hparams = p_hparams
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return model_cls(hparams, mode, p_hparams), features, hparams
def get_model(self,
hparams, mode=tf.estimator.ModeKeys.TRAIN, has_input=True):
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
hparams.mix_with_transformer = ""
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
VOCAB_SIZE,
hparams)
if not has_input:
del p_hparams.modality["inputs"]
hparams.problems = [p_hparams]
inputs = np.random.randint(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = np.random.randint(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return universal_transformer.UniversalTransformer(
hparams, mode, p_hparams), features
def testTransformer(self):
batch_size = 3
input_length = 5
target_length = 7
vocab_size = 9
hparams = transformer_revnet_test()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
hparams.problem_hparams = p_hparams
inputs = np.random.randint(vocab_size,
size=(batch_size, input_length, 1, 1))
targets = np.random.randint(vocab_size,
size=(batch_size, target_length, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = transformer_revnet.TransformerRevnet(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
grads = tf.gradients(tf.reduce_mean(logits),
[features["inputs"]] + tf.global_variables())
grads = [g for g in grads if g is not None]
with self.test_session() as session:
session.run(tf.global_variables_initializer())
logits_val, _ = session.run([logits, grads])
self.assertEqual(logits_val.shape,
(batch_size, target_length, 1, 1, vocab_size))
def testNeuralGPU(self):
hparams = common_hparams.basic_params1()
batch_size = 3
input_length = 5
target_length = input_length
input_vocab_size = 9
target_vocab_size = 11
p_hparams = problem_hparams.test_problem_hparams(hparams, input_vocab_size,
target_vocab_size)
inputs = -1 + np.random.random_integers(
input_vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
target_vocab_size, size=(batch_size, target_length, 1, 1))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32)
}
model = neural_gpu.NeuralGPU(hparams, p_hparams)
shadred_logits, _, _ = model.model_fn(features, True)
logits = tf.concat(shadred_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size, target_length, 1, 1,
target_vocab_size))
def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
has_input=False, model_cls=transformer.Transformer,
batch_size=64 ):
hparams = transformer.transformer_tall()
hparams.use_tpu = False
if hparams.get("problem_hparams", None) is None:
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
VOCAB_SIZE,
hparams)
if not has_input:
del p_hparams.modality["inputs"]
hparams.problem_hparams = p_hparams
with tf.device('cpu:0'):
inputs = np.random.randint(
VOCAB_SIZE, size=(batch_size, INPUT_LENGTH, 1, 1))
targets = np.random.randint(
VOCAB_SIZE, size=(batch_size, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return model_cls(hparams, mode, p_hparams), features
def testImagetransformer(self, net, hparams):
batch_size = 3
size = 7
vocab_size = 256
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size)
inputs = -1 + np.random.random_integers(vocab_size,
size=(batch_size, 1, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, size, size, 3))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = net(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
if hparams.likelihood == common_image_attention.DistributionType.CAT:
expected = (batch_size, size, size, 3, vocab_size)
else:
expected = (batch_size, size, size, hparams.num_mixtures * 10)
self.assertEqual(res.shape, expected)
def get_model(hparams=None, mode=tf.estimator.ModeKeys.TRAIN,
has_input=True, model_cls=transformer.Transformer):
if hparams is None:
hparams = transformer.transformer_tiny()
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE,
VOCAB_SIZE,
hparams)
if not has_input:
del p_hparams.modality["inputs"]
hparams.problem_hparams = p_hparams
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
if has_input:
features["inputs"] = tf.constant(inputs, dtype=tf.int32, name="inputs")
return model_cls(hparams, mode, p_hparams), features
def test_transformer_aux_body(self):
batch_size = 3
input_length = 5
target_length = 16
vocab_size = 9
hparams = transformer_aux.transformer_aux_tiny()
hparams.shift_values = "-5,1,2,3"
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
hparams.problem_hparams = p_hparams
inputs = np.random.randint(vocab_size,
size=(batch_size, input_length, 1, 1))
targets = np.random.randint(vocab_size,
size=(batch_size, target_length, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
tf.train.create_global_step()
model = transformer_aux.TransformerAux(hparams,
tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, losses = model(features)
self.assertIn("training", losses)
self.assertIn("auxiliary", losses)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
logits_val = session.run(logits)
self.assertEqual(logits_val.shape,
(batch_size, target_length, 1, 1, vocab_size))
def getTransformerModel(hparams, mode=tf.estimator.ModeKeys.TRAIN):
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(0, num_classes)
p_hparams.input_modality = {"inputs": ("real", 0)}
hparams.problems = [p_hparams]
return transformer.Transformer(hparams, mode)
def testByteNet(self):
vocab_size = 9
x = np.random.random_integers(1, high=vocab_size - 1, size=(3, 5, 1, 1))
y = np.random.random_integers(1, high=vocab_size - 1, size=(3, 6, 1, 1))
hparams = bytenet.bytenet_base()
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = bytenet.ByteNet(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 50, 1, 1, vocab_size))
def getModel(self):
hparams = transformer.transformer_small()
p_hparams = problem_hparams.test_problem_hparams(
hparams, VOCAB_SIZE, VOCAB_SIZE)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
return transformer.Transformer(hparams, tf.estimator.ModeKeys.PREDICT,
p_hparams), features
def _test_resnet(self, img_size, output_size):
vocab_size = 1
batch_size = 1
x = np.random.random_integers(0,
high=255,
size=(batch_size, img_size, img_size, 3))
y = np.random.random_integers(1,
high=vocab_size,
size=(batch_size, 1, 1, 1))
#hparams = resnet_tiny_cpu()
#hparams = resnet_50()
hparams = resnet_32()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
p_hparams.input_modality["inputs"] = modalities.ImageModality(hparams)
p_hparams.target_modality = modalities.ClassLabelModality(
hparams, vocab_size)
run_meta = tf.RunMetadata()
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
#model = resnet.Resnet(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
model = shake_shake.ShakeShake(hparams,
tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
print(logits.get_shape())
#opts = tf.profiler.ProfileOptionBuilder.float_operation()
#flops = tf.profiler.profile(tf.get_default_graph(), run_meta=run_meta, options=opts)
#print(flops.total_float_ops)
session.run(tf.global_variables_initializer())
#res = session.run(logits)
tf.get_variable_scope().set_initializer(
optimize.get_variable_initializer(hparams))
loss = tf.losses.sparse_softmax_cross_entropy(labels=tf.constant(
0, dtype=tf.int32, shape=[1, 1, 1, 1, 1]),
logits=logits)
train_op = optimize.optimize(loss, 0.1, hparams)
session.run(loss)
opts = tf.profiler.ProfileOptionBuilder.float_operation()
flops = tf.profiler.profile(tf.get_default_graph(),
run_meta=run_meta,
options=opts)
print(flops.total_float_ops)
def testLSTMSeq2Seq(self):
vocab_size = 9
x = np.random.random_integers(1, high=vocab_size - 1, size=(3, 5, 1, 1))
y = np.random.random_integers(1, high=vocab_size - 1, size=(3, 6, 1, 1))
hparams = common_hparams.basic_params1()
p_hparams = problem_hparams.test_problem_hparams(hparams, vocab_size,
vocab_size)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = baseline.LSTMSeq2Seq(hparams, p_hparams)
sharded_logits, _, _ = model.model_fn(features, True)
logits = tf.concat(sharded_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 6, 1, 1, vocab_size))
def testByteNet(self):
vocab_size = 9
x = np.random.randint(1, high=vocab_size, size=(3, 5, 1, 1))
y = np.random.randint(1, high=vocab_size, size=(3, 6, 1, 1))
hparams = bytenet.bytenet_base()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = bytenet.ByteNet(hparams, tf.estimator.ModeKeys.TRAIN,
p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 50, 1, 1, vocab_size))
def testLSTMSeq2seqBidirectionalEncoder(self):
vocab_size = 9
x = np.random.randint(1, high=vocab_size, size=(3, 5, 1, 1))
y = np.random.randint(1, high=vocab_size, size=(3, 6, 1, 1))
hparams = lstm.lstm_seq2seq()
p_hparams = problem_hparams.test_problem_hparams(
vocab_size, vocab_size, hparams)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = lstm.LSTMSeq2seqBidirectionalEncoder(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 6, 1, 1, vocab_size))
def testXception(self):
vocab_size = 9
x = np.random.random_integers(1, high=vocab_size - 1, size=(3, 5, 1, 1))
y = np.random.random_integers(1, high=vocab_size - 1, size=(3, 1, 1, 1))
hparams = xception.xception_base()
p_hparams = problem_hparams.test_problem_hparams(hparams, vocab_size,
vocab_size)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = xception.Xception(hparams, p_hparams)
sharded_logits, _, _ = model.model_fn(features, True)
logits = tf.concat(sharded_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 5, 1, 1, vocab_size))
def testXception(self):
vocab_size = 9
x = np.random.random_integers(1, high=vocab_size - 1, size=(3, 5, 1, 1))
y = np.random.random_integers(1, high=vocab_size - 1, size=(3, 1, 1, 1))
hparams = xception.xception_tiny()
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = xception.Xception(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
sharded_logits, _ = model.model_fn(features)
logits = tf.concat(sharded_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 5, 1, 1, vocab_size))
def get_model():
hparams = transformer.transformer_tiny()
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE, VOCAB_SIZE,
hparams)
hparams.problem_hparams = p_hparams
inputs = np.random.randint(VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = np.random.randint(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32),
"inputs": tf.constant(inputs, dtype=tf.int32, name="inputs"),
}
return (evolved_transformer.EvolvedTransformer(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams), features)
def testLSTMSeq2seqBidirectionalEncoder(self):
vocab_size = 9
x = np.random.random_integers(1, high=vocab_size - 1, size=(3, 5, 1, 1))
y = np.random.random_integers(1, high=vocab_size - 1, size=(3, 6, 1, 1))
hparams = lstm.lstm_seq2seq()
p_hparams = problem_hparams.test_problem_hparams(vocab_size,
vocab_size,
hparams)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = lstm.LSTMSeq2seqBidirectionalEncoder(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 6, 1, 1, vocab_size))
def getModel(self, hparams, mode=tf.estimator.ModeKeys.TRAIN):
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(VOCAB_SIZE, VOCAB_SIZE)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32, name="inputs"),
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
return transformer.Transformer(hparams, mode, p_hparams), features
def _testResnet(self, img_size, output_size):
vocab_size = 9
batch_size = 2
x = np.random.random_integers(
0, high=255, size=(batch_size, img_size, img_size, 3))
y = np.random.random_integers(
1, high=vocab_size - 1, size=(batch_size, 1, 1, 1))
hparams = resnet_tiny_cpu()
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
p_hparams.input_modality["inputs"] = (registry.Modalities.IMAGE, None)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = resnet.Resnet(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size,) + output_size + (1, vocab_size))
def testBlueNet(self):
vocab_size = 9
x = np.random.random_integers(1, high=vocab_size - 1, size=(3, 5, 1, 1))
y = np.random.random_integers(1, high=vocab_size - 1, size=(3, 1, 1, 1))
hparams = bluenet.bluenet_tiny()
p_hparams = problem_hparams.test_problem_hparams(hparams, vocab_size,
vocab_size)
with self.test_session() as session:
tf.train.get_or_create_global_step()
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = bluenet.BlueNet(
hparams, tf.contrib.learn.ModeKeys.TRAIN, p_hparams)
sharded_logits, _, _ = model.model_fn(features)
logits = tf.concat(sharded_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (3, 5, 1, 1, vocab_size))
def _testImagetransformer2d(self, net):
batch_size = 3
size = 7
vocab_size = 256
hparams = image_transformer_2d.imagetransformer2d_tiny()
p_hparams = problem_hparams.test_problem_hparams(vocab_size, vocab_size)
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, 1, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, size, size, 3))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = net(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size, size, size, 3, vocab_size))
def testVqaAttentionBaseline(self):
batch_size = 3
image_size = 448
vocab_size = 100
num_classes = 10
question_length = 5
answer_length = 10
x = 2 * np.random.rand(batch_size, image_size, image_size, 3) - 1
q = np.random.random_integers(
1, high=vocab_size - 1, size=(batch_size, question_length, 1, 1))
a = np.random.random_integers(
0, high=num_classes, size=(batch_size, answer_length, 1, 1))
hparams = vqa_attention.vqa_attention_base()
p_hparams = problem_hparams.test_problem_hparams(vocab_size,
vocab_size,
hparams)
p_hparams.modality["inputs"] = modalities.ImageModality(hparams)
p_hparams.modality["question"] = modalities.SymbolModality(
hparams, vocab_size)
p_hparams.modality["targets"] = modalities.MultiLabelModality(
hparams, num_classes + 1)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.float32),
"question": tf.constant(q, dtype=tf.int32),
"targets": tf.constant(a, dtype=tf.int32),
}
model = vqa_attention.VqaAttentionBaseline(
hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, losses = model(features)
session.run(tf.global_variables_initializer())
logits_, losses_ = session.run([logits, losses])
self.assertEqual(logits_.shape, (batch_size, 1, 1, 1, num_classes + 1))
self.assertEqual(losses_["training"].shape, ())
def _test_xception(self, img_size):
vocab_size = 9
batch_size = 3
x = np.random.random_integers(
0, high=255, size=(batch_size, img_size, img_size, 3))
y = np.random.random_integers(
1, high=vocab_size - 1, size=(batch_size, 1, 1, 1))
hparams = xception.xception_tiny()
p_hparams = problem_hparams.test_problem_hparams(vocab_size,
vocab_size,
hparams)
p_hparams.modality["inputs"] = modalities.ImageModality(hparams)
p_hparams.modality["targets"] = modalities.ClassLabelModality(
hparams, vocab_size)
with self.test_session() as session:
features = {
"inputs": tf.constant(x, dtype=tf.int32),
"targets": tf.constant(y, dtype=tf.int32),
}
model = xception.Xception(hparams, tf.estimator.ModeKeys.TRAIN, p_hparams)
logits, _ = model(features)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size, 1, 1, 1, vocab_size))
def _testTransformer(self, net):
batch_size = 3
input_length = 5
target_length = 7
vocab_size = 9
hparams = transformer.transformer_tiny()
p_hparams = problem_hparams.test_problem_hparams(hparams, vocab_size,
vocab_size)
inputs = -1 + np.random.random_integers(
vocab_size, size=(batch_size, input_length, 1, 1))
targets = -1 + np.random.random_integers(
vocab_size, size=(batch_size, target_length, 1, 1))
with self.test_session() as session:
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
model = net(hparams, p_hparams)
shadred_logits, _, _ = model.model_fn(features, True)
logits = tf.concat(shadred_logits, 0)
session.run(tf.global_variables_initializer())
res = session.run(logits)
self.assertEqual(res.shape, (batch_size, target_length, 1, 1, vocab_size))
