def small_librispeech_model(param_overrides=None):
hparams = transformer.transformer_small()
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = librispeech.Librispeech().get_hparams(hparams)
p_hparams.vocab_size["targets"] = VOCAB_SIZE
hparams.problem_hparams = p_hparams
model = transformer.Transformer(hparams, problem_hparams=p_hparams)
if param_overrides is not None: # Add or Set any provided HParams
assert isinstance(param_overrides, dict)
for param_name in param_overrides:
if hasattr(hparams, param_name):
hparams.set_hparam(param_name, param_overrides[param_name])
else:
hparams.add_hparam(param_name, param_overrides[param_name])
inputs = np.random.rand(BATCH_SIZE, INPUT_LENGTH, 80,
3).astype("float32") # modify for speech
targets = np.random.randint(VOCAB_SIZE,
size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.float32, name="inputs"),
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
return model, features
def getModel(self,
hparams,
mode=tf.estimator.ModeKeys.TRAIN,
has_input=True):
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(
VOCAB_SIZE, VOCAB_SIZE)
if not has_input:
p_hparams.input_modality = {}
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32, name="inputs"),
"targets": tf.constant(targets, dtype=tf.int32, name="targets"),
"target_space_id": tf.constant(1, dtype=tf.int32)
}
return transformer.Transformer(hparams, mode, p_hparams), features
def getTransformerModel(hparams, mode=tf.estimator.ModeKeys.TRAIN):
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(0, num_classes)
p_hparams.input_modality = {"inputs": ("real", 0)}
hparams.problems = [p_hparams]
return transformer.Transformer(hparams, mode)
def getModel(self):
hparams = transformer.transformer_small()
p_hparams = problem_hparams.test_problem_hparams(
hparams, VOCAB_SIZE, VOCAB_SIZE)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
return transformer.Transformer(hparams, tf.estimator.ModeKeys.PREDICT,
p_hparams), features
def testTransformerWithoutProblem(self):
hparams = transformer.transformer_test()
embedded_inputs = np.random.random_sample(
(BATCH_SIZE, INPUT_LENGTH, 1, hparams.hidden_size))
embedded_targets = np.random.random_sample(
(BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size))
transformed_features = {
"inputs": tf.constant(embedded_inputs, dtype=tf.float32),
"targets": tf.constant(embedded_targets, dtype=tf.float32)
}
model = transformer.Transformer(hparams)
body_out, _ = model(transformed_features)
self.assertAllEqual(
body_out.get_shape().as_list(),
[BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size])
def encode(self, indexed_batch, mask):
"""Take in a batch of encoded sentences formed of word indices, return a batch of sentence vectors."""
word_embeddings = tf.get_variable(
name='word_embeddings',
shape=[FLAGS.vocab_size, FLAGS.wembedding_size],
initializer=tf.random_uniform_initializer(),
trainable=True)
# Mask the padded word embeddings
words = tf.nn.embedding_lookup(word_embeddings, indexed_batch)
words = tf.multiply(
words, tf.cast(tf.expand_dims(mask, -1), dtype=tf.float32))
words = tf.expand_dims(words, 1)
transformer_params = transformer.transformer_big()
transformer_params.num_heads = 5
transformer_params.hidden_size = FLAGS.wembedding_size
# Transformer encoder outputs shape [BatchSize MaxLength HiddenSize]
tfmr = transformer.Transformer(transformer_params,
mode=tf.estimator.ModeKeys.TRAIN)
target_space_id = tf.constant(1, dtype=tf.int32)
encoder_output, _ = tfmr.encode(words, target_space_id,
transformer_params)
# Use a linear transform to map onto shape [BatchSize, SentenceEmbeddingSize]
encoder_output = tf.reshape(encoder_output, [FLAGS.batch_size, -1])
matrix_shape = [
FLAGS.wembedding_size * FLAGS.max_sentence_length,
FLAGS.embedding_size
]
matrix = tf.random_normal(matrix_shape,
dtype=tf.float32,
name='linear_layer')
linear_transform = tf.Variable(matrix)
sentence_embeddings = tf.matmul(encoder_output, linear_transform)
return sentence_embeddings
def getModel(self, mode=tf.estimator.ModeKeys.TRAIN):
hparams = transformer.transformer_small()
hparams.hidden_size = 8
hparams.filter_size = 32
hparams.num_heads = 1
hparams.layer_prepostprocess_dropout = 0.0
p_hparams = problem_hparams.test_problem_hparams(
VOCAB_SIZE, VOCAB_SIZE)
hparams.problems = [p_hparams]
inputs = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1))
targets = -1 + np.random.random_integers(
VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1))
features = {
"inputs": tf.constant(inputs, dtype=tf.int32),
"targets": tf.constant(targets, dtype=tf.int32),
"target_space_id": tf.constant(1, dtype=tf.int32),
}
return transformer.Transformer(hparams, tf.estimator.ModeKeys.PREDICT,
p_hparams), features
def getModel(self, hparams, p_hparams):
return transformer.Transformer(
hparams, tf.estimator.ModeKeys.PREDICT, p_hparams)
for i in range(0, len(x), batch_size):
if (len(x[i:(i + batch_size)]) == batch_size):
yield x[i:(i + batch_size)], y[i:(i + batch_size)]
#create network
nn_graph = tf.Graph()
with nn_graph.as_default():
_inputs = tf.placeholder(tf.int32, [None, None], name = 'inputs')
_labels = tf.placeholder(tf.int32, [None, None], name = 'labels')
keep_prob = tf.placeholder(tf.float32, name = 'keep_prob')
embedding = tf.Variable(tf.random_uniform((n_words, embed_size), \
minval = -1, maxval = 1))
embed = tf.nn.embedding_lookup(embedding, _inputs) #embeddings
model = transformer.Transformer(hparams)
decoder = model.decode(embed, None, None, _inputs, hparams)
print(decoder)
exit()
#as opposed to None, use empty encoding vars
# - encode an empty sequence = model encoder doing nothing
outputs, _ = tf.contrib.seq2seq.dynamic_decode(decoder)
logits = outputs.rnn_output
predictions = tf.contrib.layers.fully_connected(outputs[:, -1], 1, \
activation_fn = tf.sigmoid)
cost = tf.losses.mean_squared_error(_labels, predictions)
tf.summary.scalar("loss", cost)
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
def _init_env(self):
FLAGS.use_tpu = False
tf.logging.set_verbosity(tf.logging.DEBUG)
tf.logging.info("Import usr dir from %s", self._usr_dir)
if self._usr_dir != None:
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
tf.logging.info("Start to create hparams,for %s of %s", self._problem,
self._hparams_set)
self._hparams = create_hparams()
self._hparams_decode = create_decode_hparams(
extra_length=self._extra_length,
batch_size=self._batch_size,
beam_size=self._beam_size,
alpha=self._alpha,
return_beams=self._return_beams,
write_beam_scores=self._write_beam_scores)
# self.estimator_spec = t2t_model.T2TModel.make_estimator_model_fn(
# self._model_name, self._hparams, decode_hparams=self._hparams_decode, use_tpu=False)
self.estimator = trainer_lib.create_estimator(
FLAGS.model,
self._hparams,
t2t_trainer.create_run_config(self._hparams),
decode_hparams=self._hparams_decode,
use_tpu=False)
tf.logging.info("Finish intialize environment")
#######
### make input placeholder
self._inputs_ph = tf.placeholder(
dtype=tf.int32) # shape not specified,any shape
x = tf.placeholder(dtype=tf.int32)
x.set_shape([None, None]) # ? -> (?,?)
x = tf.expand_dims(x, axis=[2]) # -> (?,?,1)
x = tf.to_int32(x)
self._inputs_ph = x
#batch_inputs = tf.reshape(self._inputs_ph, [self._batch_size, -1, 1, 1])
batch_inputs = x
# batch_inputs = tf.reshape(self._inputs_ph, [-1, -1, 1, 1])
#targets_ph = tf.placeholder(dtype=tf.int32)
#batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1])
self._features = {
"inputs": batch_inputs,
"problem_choice": 0, # We run on the first problem here.
"input_space_id": self._hparams.problems[0].input_space_id,
"target_space_id": self._hparams.problems[0].target_space_id
}
### 加入 decode length 变长的
self.input_extra_length_ph = tf.placeholder(dtype=tf.int32)
self._features['decode_length'] = self.input_extra_length_ph
## 去掉 整数的
del self._features["problem_choice"]
del self._features["input_space_id"]
del self._features["target_space_id"]
####
mode = tf.estimator.ModeKeys.PREDICT
# estimator_spec = model_builder.model_fn(self._model_name, features, mode, self._hparams,
# problem_names=[self._problem], decode_hparams=self._hparams_dc)
######
from tensor2tensor.models import transformer
model_i = transformer.Transformer(hparams=self._hparams,
mode=mode,
decode_hparams=self._hparams_decode)
#problem_hparams=p_hparams,
# self._beam_result = model_i._fast_decode(self._features, decode_length=5, beam_size=10, top_beams=10,
# alpha=0.6) #fail
self._beam_result = model_i._beam_decode(self._features,
decode_length=5,
beam_size=self._beam_size,
top_beams=self._beam_size,
alpha=0.6)
"""
######
predictions_dict = self.estimator._call_model_fn(self._features,None,mode,t2t_trainer.create_run_config(self._hparams))
self._predictions_dict=predictions_dict.predictions
"""
#self._predictions = self._predictions_dict["outputs"]
# self._scores=predictions_dict['scores'] not return when greedy search
tf.logging.info("Start to init tf session")
if self._isGpu:
print('Using GPU in Decoder')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=self._fraction)
self._sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=gpu_options))
else:
print('Using CPU in Decoder')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0)
config = tf.ConfigProto(gpu_options=gpu_options)
config.allow_soft_placement = True
config.log_device_placement = False
self._sess = tf.Session(config=config)
with self._sess.as_default():
ckpt = saver_mod.get_checkpoint_state(self._model_dir)
saver = tf.train.Saver()
tf.logging.info("Start to restore the parameters from %s",
ckpt.model_checkpoint_path)
saver.restore(self._sess, ckpt.model_checkpoint_path)
tf.logging.info("Finish intialize environment")
