def create_load_vocab(arg, file_name, out_file_name, pad=True, unk=True, sos_eos=False): """Creates and loads the vocab file for a given corpus. Args: arg: The output of the parser. file_name: The name of the file containing the corpus. out_file_name: The file into which the vocab should be written into. pad: A boolean to indicate if the pad token should be included in the vocabulary. unk: A boolean to indicate if the unknown token should be included in the vocabulary. sos_eos: A boolean to indicate if the SOS and EOS token should be included in the vocabulary. Returns: A dictionary of the vocabulary and it's corresponding index. It also includes a list of all the vocabulary. """ full_path = os.path.join('./top_data', arg.train_data_path, file_name) output_path = os.path.join(arg.vocab_path, out_file_name) create_vocabulary(full_path, output_path, pad, unk, sos_eos) vocab = load_vocabulary(output_path) return vocab
def __init__(self, train_file=None, valid_file=None, test_file=None): """Create G2P model and initialize or load parameters in session.""" self.test_file = test_file # Preliminary actions before model creation. if FLAGS.train: #Load model parameters. num_layers, size = data_utils.save_params(FLAGS.num_layers, FLAGS.size, FLAGS.model) batch_size = FLAGS.batch_size # Prepare G2P data. print("Preparing G2P data") train_gr_ids, train_ph_ids, valid_gr_ids, valid_ph_ids, self.gr_vocab,\ self.ph_vocab = data_utils.prepare_g2p_data(FLAGS.model, train_file, valid_file) # Read data into buckets and compute their sizes. print ("Reading development and training data.") self.valid_set = self.__put_into_buckets(valid_gr_ids, valid_ph_ids) self.train_set = self.__put_into_buckets(train_gr_ids, train_ph_ids) else: #Load model parameters. num_layers, size = data_utils.load_params(FLAGS.num_layers, FLAGS.size, FLAGS.model) batch_size = 1 # We decode one word at a time. # Load vocabularies self.gr_vocab = data_utils.load_vocabulary(os.path.join(FLAGS.model, "vocab.grapheme")) self.ph_vocab = data_utils.load_vocabulary(os.path.join(FLAGS.model, "vocab.phoneme")) self.rev_ph_vocab =\ data_utils.load_vocabulary(os.path.join(FLAGS.model, "vocab.phoneme"), reverse=True) self.session = tf.Session() # Create model. print("Creating %d layers of %d units." % (num_layers, size)) self.model = seq2seq_model.Seq2SeqModel(len(self.gr_vocab), len(self.ph_vocab), self._BUCKETS, size, num_layers, FLAGS.max_gradient_norm, batch_size, FLAGS.learning_rate, FLAGS.learning_rate_decay_factor, forward_only=not FLAGS.train) self.model.saver = tf.train.Saver(tf.all_variables(), max_to_keep=1) self.__create_model()
def __init__(self, train_dic=None, valid_dic=None, test_dic=None): """Create G2P model and initialize or load parameters in session.""" self.test_dic = test_dic # Preliminary actions before model creation. if FLAGS.train: #Load model parameters. num_layers, size = data_utils.save_params(FLAGS.num_layers, FLAGS.size, FLAGS.model) batch_size = FLAGS.batch_size # Prepare G2P data. print("Preparing G2P data") train_gr_ids, train_ph_ids, valid_gr_ids, valid_ph_ids, self.gr_vocab,\ self.ph_vocab = data_utils.prepare_g2p_data(FLAGS.model, train_dic, valid_dic) # Read data into buckets and compute their sizes. print ("Reading development and training data.") self.valid_set = self.__put_into_buckets(valid_gr_ids, valid_ph_ids) self.train_set = self.__put_into_buckets(train_gr_ids, train_ph_ids) else: #Load model parameters. num_layers, size = data_utils.load_params(FLAGS.num_layers, FLAGS.size, FLAGS.model) batch_size = 1 # We decode one word at a time. # Load vocabularies self.gr_vocab = data_utils.load_vocabulary(os.path.join(FLAGS.model, "vocab.grapheme")) self.ph_vocab = data_utils.load_vocabulary(os.path.join(FLAGS.model, "vocab.phoneme")) self.rev_ph_vocab =\ data_utils.load_vocabulary(os.path.join(FLAGS.model, "vocab.phoneme"), reverse=True) self.session = tf.Session() # Create model. print("Creating %d layers of %d units." % (num_layers, size)) self.model = seq2seq_model.Seq2SeqModel(len(self.gr_vocab), len(self.ph_vocab), self._BUCKETS, size, num_layers, FLAGS.max_gradient_norm, batch_size, FLAGS.learning_rate, FLAGS.learning_rate_decay_factor, forward_only=not FLAGS.train) self.__create_model()
def load_vocabs_load_model(sess): """Load vocabularies and saved model. Returns: gr_vocab: Grapheme vocabulary; rev_ph_vocab: Reversed phoneme vocabulary; model: Trained model. """ # Load vocabularies gr_vocab_path = os.path.join(FLAGS.model, "vocab.grapheme") ph_vocab_path = os.path.join(FLAGS.model, "vocab.phoneme") gr_vocab = data_utils.load_vocabulary(gr_vocab_path, False) rev_ph_vocab = data_utils.load_vocabulary(ph_vocab_path, True) # Get vocabulary sizes gr_vocab_size = len(gr_vocab) ph_vocab_size = len(rev_ph_vocab) # Load model model = create_model(sess, True, gr_vocab_size, ph_vocab_size) model.batch_size = 1 # We decode one word at a time. return (gr_vocab, rev_ph_vocab, model)
def load_vocabs_load_model(sess): """Load vocabularies and saved model. Returns: gr_vocab: Grapheme vocabulary; rev_ph_vocab: Reversed phoneme vocabulary; model: Trained model. """ # Load vocabularies gr_vocab_path = os.path.join(FLAGS.model, "vocab.grapheme") ph_vocab_path = os.path.join(FLAGS.model, "vocab.phoneme") gr_vocab = data_utils.load_vocabulary(gr_vocab_path, False) rev_ph_vocab = data_utils.load_vocabulary(ph_vocab_path, True) # Get vocabulary sizes gr_vocab_size = len(gr_vocab) ph_vocab_size = len(rev_ph_vocab) # Load model model = create_model(sess, True, gr_vocab_size, ph_vocab_size) model.batch_size = 1 # We decode one word at a time. return (gr_vocab, rev_ph_vocab, model)
def load_decode_model(self): """Load G2P model and initialize or load parameters in session.""" if not os.path.exists(os.path.join(self.model_dir, 'checkpoint')): raise RuntimeError("Model not found in %s" % self.model_dir) self.batch_size = 1 # We decode one word at a time. #Load model parameters. num_layers, size = data_utils.load_params(self.model_dir) # Load vocabularies print("Loading vocabularies from %s" % self.model_dir) self.gr_vocab = data_utils.load_vocabulary( os.path.join(self.model_dir, "vocab.grapheme")) self.ph_vocab = data_utils.load_vocabulary( os.path.join(self.model_dir, "vocab.phoneme")) self.rev_ph_vocab =\ data_utils.load_vocabulary(os.path.join(self.model_dir, "vocab.phoneme"), reverse=True) self.session = tf.Session() # Restore model. print("Creating %d layers of %d units." % (num_layers, size)) self.model = seq2seq_model.Seq2SeqModel(len(self.gr_vocab), len(self.ph_vocab), self._BUCKETS, size, num_layers, 0, self.batch_size, 0, 0, forward_only=True) self.model.saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) # Check for saved models and restore them. print("Reading model parameters from %s" % self.model_dir) self.model.saver.restore(self.session, os.path.join(self.model_dir, "model"))
def train(): embedding_mat = data_utils.load_embedding_mat(FLAGS.save_embedding_file) vocab = data_utils.load_vocabulary(FLAGS.vocabulary_file) model = cnn_model.CNN_MODEL(embed_dim=FLAGS.embedding_dim, filter_sizes=FLAGS.filter_sizes, max_sent_len=FLAGS.max_sentence_len, embedding_mat=embedding_mat, word_nums=len(vocab), filter_nums=FLAGS.filter_nums, label_nums=FLAGS.label_nums, learning_rate=FLAGS.learning_rate, model_path=FLAGS.model_path, epoch=FLAGS.num_epochs, batch_size=FLAGS.batch_size, dropout_prob=FLAGS.dropout_keep_prob) train_data = data_utils.generate_data('./data/train_data.ids', FLAGS.max_sentence_len, vocab) valid_data = data_utils.generate_data('./data/valid_data.ids', FLAGS.max_sentence_len, vocab) print('train data size is {}, valid data size is {}.'.format( len(train_data[0]), len(valid_data[0]))) model.train(train_data, valid_data)
def train(train_dic, valid_dic, test_dic): """Train a gr->ph translation model using G2P data.""" if not os.path.exists(FLAGS.model): os.makedirs(FLAGS.model) # Save model's architecture params_path = os.path.join(FLAGS.model, "model.params") with open(params_path, 'w') as param_file: param_file.write("num_layers:" + str(FLAGS.num_layers) + "\n") param_file.write("size:" + str(FLAGS.size)) # Prepare G2P data. print("Preparing G2P data") train_gr, train_ph = data_utils.split_to_grapheme_phoneme(train_dic) valid_gr, valid_ph = data_utils.split_to_grapheme_phoneme(valid_dic) train_gr_ids, train_ph_ids, valid_gr_ids, valid_ph_ids, gr_vocab, ph_vocab =\ data_utils.prepare_g2p_data(FLAGS.model, train_gr, train_ph, valid_gr, valid_ph) gr_vocab_size = len(gr_vocab) ph_vocab_size = len(ph_vocab) with tf.Session() as sess: # Create model. print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.size)) model = create_model(sess, False, gr_vocab_size, ph_vocab_size) # Read data into buckets and compute their sizes. print ("Reading development and training data.") valid_set = put_into_buckets(valid_gr_ids, valid_ph_ids) train_set = put_into_buckets(train_gr_ids, train_ph_ids) train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_BUCKETS))] train_total_size = float(sum(train_bucket_sizes)) # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to # the size if i-th training bucket, as used later. train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))] # This is the training loop. step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] while (FLAGS.max_steps == 0 or model.global_step.eval() <= FLAGS.max_steps): # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01]) # Get a batch and make a step. start_time = time.time() encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id) _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, False) step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint loss += step_loss / FLAGS.steps_per_checkpoint current_step += 1 # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % FLAGS.steps_per_checkpoint == 0: # Print statistics for the previous epoch. perplexity = math.exp(loss) if loss < 300 else float('inf') print ("global step %d learning rate %.4f step-time %.2f perplexity " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, perplexity)) # Decrease learning rate if no improvement was seen over last 3 times. if len(previous_losses) > 2 and loss > max(previous_losses[-3:]): sess.run(model.learning_rate_decay_op) if len(previous_losses) > 34 and \ previous_losses[-35:-34] <= min(previous_losses[-35:]): break previous_losses.append(loss) # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join(FLAGS.model, "translate.ckpt") model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 # Run evals on development set and print their perplexity. for bucket_id in xrange(len(_BUCKETS)): encoder_inputs, decoder_inputs, target_weights = model.get_batch( valid_set, bucket_id) _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True) eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf') print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx)) print('Training process stopped.') print('Beginning calculation word error rate (WER) on test sample.') ph_vocab_path = os.path.join(FLAGS.model, "vocab.phoneme") rev_ph_vocab = data_utils.load_vocabulary(ph_vocab_path, True) model.forward_only = True model.batch_size = 1 # We decode one word at a time. evaluate(test_dic, sess, model, gr_vocab, rev_ph_vocab, ph_vocab)
def build_vocabularies(): vocabulary = du.load_vocabulary() rev_vocabulary = du.build_reverse_vocabulary(vocabulary) return (vocabulary, rev_vocabulary)
def train(train_dic, valid_dic, test_dic): """Train a gr->ph translation model using G2P data.""" if not os.path.exists(FLAGS.model): os.makedirs(FLAGS.model) # Save model's architecture params_path = os.path.join(FLAGS.model, "model.params") with open(params_path, 'w') as param_file: param_file.write("num_layers:" + str(FLAGS.num_layers) + "\n") param_file.write("size:" + str(FLAGS.size)) # Prepare G2P data. print("Preparing G2P data") train_gr, train_ph = data_utils.split_to_grapheme_phoneme(train_dic) valid_gr, valid_ph = data_utils.split_to_grapheme_phoneme(valid_dic) train_gr_ids, train_ph_ids, valid_gr_ids, valid_ph_ids, gr_vocab, ph_vocab =\ data_utils.prepare_g2p_data(FLAGS.model, train_gr, train_ph, valid_gr, valid_ph) gr_vocab_size = len(gr_vocab) ph_vocab_size = len(ph_vocab) with tf.Session() as sess: # Create model. print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.size)) model = create_model(sess, False, gr_vocab_size, ph_vocab_size) # Read data into buckets and compute their sizes. print("Reading development and training data.") valid_set = put_into_buckets(valid_gr_ids, valid_ph_ids) train_set = put_into_buckets(train_gr_ids, train_ph_ids) train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_BUCKETS))] train_total_size = float(sum(train_bucket_sizes)) # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to # the size if i-th training bucket, as used later. train_buckets_scale = [ sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes)) ] # This is the training loop. step_time, loss = 0.0, 0.0 current_step = 0 previous_losses = [] while (FLAGS.max_steps == 0 or model.global_step.eval() <= FLAGS.max_steps): # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([ i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01 ]) # Get a batch and make a step. start_time = time.time() encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id) _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, False) step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint loss += step_loss / FLAGS.steps_per_checkpoint current_step += 1 # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % FLAGS.steps_per_checkpoint == 0: # Print statistics for the previous epoch. perplexity = math.exp(loss) if loss < 300 else float('inf') print( "global step %d learning rate %.4f step-time %.2f perplexity " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, perplexity)) # Decrease learning rate if no improvement was seen over last 3 times. if len(previous_losses) > 2 and loss > max( previous_losses[-3:]): sess.run(model.learning_rate_decay_op) if len(previous_losses) > 34 and \ previous_losses[-35:-34] <= min(previous_losses[-35:]): break previous_losses.append(loss) # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join(FLAGS.model, "translate.ckpt") model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 # Run evals on development set and print their perplexity. for bucket_id in xrange(len(_BUCKETS)): encoder_inputs, decoder_inputs, target_weights = model.get_batch( valid_set, bucket_id) _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True) eval_ppx = math.exp( eval_loss) if eval_loss < 300 else float('inf') print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx)) print('Training process stopped.') print('Beginning calculation word error rate (WER) on test sample.') ph_vocab_path = os.path.join(FLAGS.model, "vocab.phoneme") rev_ph_vocab = data_utils.load_vocabulary(ph_vocab_path, True) model.forward_only = True model.batch_size = 1 # We decode one word at a time. evaluate(test_dic, sess, model, gr_vocab, rev_ph_vocab, ph_vocab)