def small_librispeech_model(param_overrides=None): hparams = transformer.transformer_small() hparams.hidden_size = 8 hparams.filter_size = 32 hparams.num_heads = 1 hparams.layer_prepostprocess_dropout = 0.0 p_hparams = librispeech.Librispeech().get_hparams(hparams) p_hparams.vocab_size["targets"] = VOCAB_SIZE hparams.problem_hparams = p_hparams model = transformer.Transformer(hparams, problem_hparams=p_hparams) if param_overrides is not None: # Add or Set any provided HParams assert isinstance(param_overrides, dict) for param_name in param_overrides: if hasattr(hparams, param_name): hparams.set_hparam(param_name, param_overrides[param_name]) else: hparams.add_hparam(param_name, param_overrides[param_name]) inputs = np.random.rand(BATCH_SIZE, INPUT_LENGTH, 80, 3).astype("float32") # modify for speech targets = np.random.randint(VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1)) features = { "inputs": tf.constant(inputs, dtype=tf.float32, name="inputs"), "targets": tf.constant(targets, dtype=tf.int32, name="targets"), "target_space_id": tf.constant(1, dtype=tf.int32) } return model, features
def getModel(self, hparams, mode=tf.estimator.ModeKeys.TRAIN, has_input=True): hparams.hidden_size = 8 hparams.filter_size = 32 hparams.num_heads = 1 hparams.layer_prepostprocess_dropout = 0.0 p_hparams = problem_hparams.test_problem_hparams( VOCAB_SIZE, VOCAB_SIZE) if not has_input: p_hparams.input_modality = {} hparams.problems = [p_hparams] inputs = -1 + np.random.random_integers( VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1)) targets = -1 + np.random.random_integers( VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1)) features = { "inputs": tf.constant(inputs, dtype=tf.int32, name="inputs"), "targets": tf.constant(targets, dtype=tf.int32, name="targets"), "target_space_id": tf.constant(1, dtype=tf.int32) } return transformer.Transformer(hparams, mode, p_hparams), features
def getTransformerModel(hparams, mode=tf.estimator.ModeKeys.TRAIN): hparams.hidden_size = 8 hparams.filter_size = 32 hparams.num_heads = 1 hparams.layer_prepostprocess_dropout = 0.0 p_hparams = problem_hparams.test_problem_hparams(0, num_classes) p_hparams.input_modality = {"inputs": ("real", 0)} hparams.problems = [p_hparams] return transformer.Transformer(hparams, mode)
def getModel(self): hparams = transformer.transformer_small() p_hparams = problem_hparams.test_problem_hparams( hparams, VOCAB_SIZE, VOCAB_SIZE) hparams.problems = [p_hparams] inputs = -1 + np.random.random_integers( VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1)) targets = -1 + np.random.random_integers( VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1)) features = { "inputs": tf.constant(inputs, dtype=tf.int32), "targets": tf.constant(targets, dtype=tf.int32), "target_space_id": tf.constant(1, dtype=tf.int32), } return transformer.Transformer(hparams, tf.estimator.ModeKeys.PREDICT, p_hparams), features
def testTransformerWithoutProblem(self): hparams = transformer.transformer_test() embedded_inputs = np.random.random_sample( (BATCH_SIZE, INPUT_LENGTH, 1, hparams.hidden_size)) embedded_targets = np.random.random_sample( (BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size)) transformed_features = { "inputs": tf.constant(embedded_inputs, dtype=tf.float32), "targets": tf.constant(embedded_targets, dtype=tf.float32) } model = transformer.Transformer(hparams) body_out, _ = model(transformed_features) self.assertAllEqual( body_out.get_shape().as_list(), [BATCH_SIZE, TARGET_LENGTH, 1, hparams.hidden_size])
def encode(self, indexed_batch, mask): """Take in a batch of encoded sentences formed of word indices, return a batch of sentence vectors.""" word_embeddings = tf.get_variable( name='word_embeddings', shape=[FLAGS.vocab_size, FLAGS.wembedding_size], initializer=tf.random_uniform_initializer(), trainable=True) # Mask the padded word embeddings words = tf.nn.embedding_lookup(word_embeddings, indexed_batch) words = tf.multiply( words, tf.cast(tf.expand_dims(mask, -1), dtype=tf.float32)) words = tf.expand_dims(words, 1) transformer_params = transformer.transformer_big() transformer_params.num_heads = 5 transformer_params.hidden_size = FLAGS.wembedding_size # Transformer encoder outputs shape [BatchSize MaxLength HiddenSize] tfmr = transformer.Transformer(transformer_params, mode=tf.estimator.ModeKeys.TRAIN) target_space_id = tf.constant(1, dtype=tf.int32) encoder_output, _ = tfmr.encode(words, target_space_id, transformer_params) # Use a linear transform to map onto shape [BatchSize, SentenceEmbeddingSize] encoder_output = tf.reshape(encoder_output, [FLAGS.batch_size, -1]) matrix_shape = [ FLAGS.wembedding_size * FLAGS.max_sentence_length, FLAGS.embedding_size ] matrix = tf.random_normal(matrix_shape, dtype=tf.float32, name='linear_layer') linear_transform = tf.Variable(matrix) sentence_embeddings = tf.matmul(encoder_output, linear_transform) return sentence_embeddings
def getModel(self, mode=tf.estimator.ModeKeys.TRAIN): hparams = transformer.transformer_small() hparams.hidden_size = 8 hparams.filter_size = 32 hparams.num_heads = 1 hparams.layer_prepostprocess_dropout = 0.0 p_hparams = problem_hparams.test_problem_hparams( VOCAB_SIZE, VOCAB_SIZE) hparams.problems = [p_hparams] inputs = -1 + np.random.random_integers( VOCAB_SIZE, size=(BATCH_SIZE, INPUT_LENGTH, 1, 1)) targets = -1 + np.random.random_integers( VOCAB_SIZE, size=(BATCH_SIZE, TARGET_LENGTH, 1, 1)) features = { "inputs": tf.constant(inputs, dtype=tf.int32), "targets": tf.constant(targets, dtype=tf.int32), "target_space_id": tf.constant(1, dtype=tf.int32), } return transformer.Transformer(hparams, tf.estimator.ModeKeys.PREDICT, p_hparams), features
def getModel(self, hparams, p_hparams): return transformer.Transformer( hparams, tf.estimator.ModeKeys.PREDICT, p_hparams)
for i in range(0, len(x), batch_size): if (len(x[i:(i + batch_size)]) == batch_size): yield x[i:(i + batch_size)], y[i:(i + batch_size)] #create network nn_graph = tf.Graph() with nn_graph.as_default(): _inputs = tf.placeholder(tf.int32, [None, None], name = 'inputs') _labels = tf.placeholder(tf.int32, [None, None], name = 'labels') keep_prob = tf.placeholder(tf.float32, name = 'keep_prob') embedding = tf.Variable(tf.random_uniform((n_words, embed_size), \ minval = -1, maxval = 1)) embed = tf.nn.embedding_lookup(embedding, _inputs) #embeddings model = transformer.Transformer(hparams) decoder = model.decode(embed, None, None, _inputs, hparams) print(decoder) exit() #as opposed to None, use empty encoding vars # - encode an empty sequence = model encoder doing nothing outputs, _ = tf.contrib.seq2seq.dynamic_decode(decoder) logits = outputs.rnn_output predictions = tf.contrib.layers.fully_connected(outputs[:, -1], 1, \ activation_fn = tf.sigmoid) cost = tf.losses.mean_squared_error(_labels, predictions) tf.summary.scalar("loss", cost) optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
def _init_env(self): FLAGS.use_tpu = False tf.logging.set_verbosity(tf.logging.DEBUG) tf.logging.info("Import usr dir from %s", self._usr_dir) if self._usr_dir != None: usr_dir.import_usr_dir(FLAGS.t2t_usr_dir) tf.logging.info("Start to create hparams,for %s of %s", self._problem, self._hparams_set) self._hparams = create_hparams() self._hparams_decode = create_decode_hparams( extra_length=self._extra_length, batch_size=self._batch_size, beam_size=self._beam_size, alpha=self._alpha, return_beams=self._return_beams, write_beam_scores=self._write_beam_scores) # self.estimator_spec = t2t_model.T2TModel.make_estimator_model_fn( # self._model_name, self._hparams, decode_hparams=self._hparams_decode, use_tpu=False) self.estimator = trainer_lib.create_estimator( FLAGS.model, self._hparams, t2t_trainer.create_run_config(self._hparams), decode_hparams=self._hparams_decode, use_tpu=False) tf.logging.info("Finish intialize environment") ####### ### make input placeholder self._inputs_ph = tf.placeholder( dtype=tf.int32) # shape not specified,any shape x = tf.placeholder(dtype=tf.int32) x.set_shape([None, None]) # ? -> (?,?) x = tf.expand_dims(x, axis=[2]) # -> (?,?,1) x = tf.to_int32(x) self._inputs_ph = x #batch_inputs = tf.reshape(self._inputs_ph, [self._batch_size, -1, 1, 1]) batch_inputs = x # batch_inputs = tf.reshape(self._inputs_ph, [-1, -1, 1, 1]) #targets_ph = tf.placeholder(dtype=tf.int32) #batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1]) self._features = { "inputs": batch_inputs, "problem_choice": 0, # We run on the first problem here. "input_space_id": self._hparams.problems[0].input_space_id, "target_space_id": self._hparams.problems[0].target_space_id } ### 加入 decode length 变长的 self.input_extra_length_ph = tf.placeholder(dtype=tf.int32) self._features['decode_length'] = self.input_extra_length_ph ## 去掉 整数的 del self._features["problem_choice"] del self._features["input_space_id"] del self._features["target_space_id"] #### mode = tf.estimator.ModeKeys.PREDICT # estimator_spec = model_builder.model_fn(self._model_name, features, mode, self._hparams, # problem_names=[self._problem], decode_hparams=self._hparams_dc) ###### from tensor2tensor.models import transformer model_i = transformer.Transformer(hparams=self._hparams, mode=mode, decode_hparams=self._hparams_decode) #problem_hparams=p_hparams, # self._beam_result = model_i._fast_decode(self._features, decode_length=5, beam_size=10, top_beams=10, # alpha=0.6) #fail self._beam_result = model_i._beam_decode(self._features, decode_length=5, beam_size=self._beam_size, top_beams=self._beam_size, alpha=0.6) """ ###### predictions_dict = self.estimator._call_model_fn(self._features,None,mode,t2t_trainer.create_run_config(self._hparams)) self._predictions_dict=predictions_dict.predictions """ #self._predictions = self._predictions_dict["outputs"] # self._scores=predictions_dict['scores'] not return when greedy search tf.logging.info("Start to init tf session") if self._isGpu: print('Using GPU in Decoder') gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=self._fraction) self._sess = tf.Session( config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options)) else: print('Using CPU in Decoder') gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0) config = tf.ConfigProto(gpu_options=gpu_options) config.allow_soft_placement = True config.log_device_placement = False self._sess = tf.Session(config=config) with self._sess.as_default(): ckpt = saver_mod.get_checkpoint_state(self._model_dir) saver = tf.train.Saver() tf.logging.info("Start to restore the parameters from %s", ckpt.model_checkpoint_path) saver.restore(self._sess, ckpt.model_checkpoint_path) tf.logging.info("Finish intialize environment")