def main(): g = Graph() print("Training Graph loaded") with g.graph.as_default(): # Load vocabulary char2idx, idx2char = load_vocab() # Training sv = tf.train.Supervisor(logdir=hp.logdir, save_model_secs=0) with sv.managed_session() as sess: for epoch in range(1, hp.num_epochs + 1): if sv.should_stop(): break for step in tqdm(range(g.num_batch), total=g.num_batch, ncols=70, leave=False, unit='b'): sess.run(g.train_op) # Write checkpoint files at every epoch gs = sess.run(g.global_step) sv.saver.save( sess, hp.logdir + '/model_epoch_%02d_gs_%d' % (epoch, gs))
def main(): g = ModelGraph(mode="test") with tf.Session() as sess: tf.sg_init(sess) # restore parameters saver = tf.train.Saver() if Hyperparams.isqwerty: save_path = "qwerty/asset/train/ckpt" else: save_path = "nine/asset/train/ckpt" saver.restore(sess, tf.train.latest_checkpoint(save_path)) mname = open(save_path + "/checkpoint", 'r').read().split('"')[1] nums, X, expected_list = load_test_data() pnyn2idx, idx2pnyn, hanzi2idx, idx2hanzi = load_vocab() with codecs.open('data/output_{}.txt'.format(mname), 'w', 'utf-8') as fout: cum_score = 0 full_score = 0 for step in range(len(X) // 64 + 1): n = nums[step * 64:(step + 1) * 64] #number batch x = X[step * 64:(step + 1) * 64] # input batch e = expected_list[step * 64:(step + 1) * 64] # batch of ground truth strings # predict characters logits = sess.run(g.logits, {g.x: x}) preds = np.squeeze(np.argmax(logits, -1)) for nn, xx, pp, ee in zip(n, x, preds, e): # sentence-wise got = '' for xxx, ppp in zip(xx, pp): # character-wise if xxx == 0: break if xxx == 1 or ppp == 1: got += "*" else: got += idx2hanzi.get(ppp, "*") got = got.replace("_", "") # Remove blanks error = distance.levenshtein(ee, got) score = len(ee) - error cum_score += score full_score += len(ee) fout.write(u"{}\t{}\t{}\t{}\n".format(nn, ee, got, score)) fout.write(u"Total acc.: {}/{}={}\n".format( cum_score, full_score, round(float(cum_score) / full_score, 2)))
def encode(inputs, is_training=True, scope="encoder", reuse=None): ''' Args: inputs: A 2d tensor with shape of [N, T], dtype of int32. seqlens: A 1d tensor with shape of [N,], dtype of int32. masks: A 3d tensor with shape of [N, T, 1], dtype of float32. is_training: Whether or not the layer is in training mode. scope: Optional scope for `variable_scope` reuse: Boolean, whether to reuse the weights of a previous layer by the same name. Returns: A collection of Hidden vectors, whose shape is (N, T, E). ''' with tf.variable_scope(scope, reuse=reuse): # Load vocabulary char2idx, idx2char = load_vocab() # Character Embedding inputs = embed(inputs, len(char2idx), hp.embed_size) # (N, T, E) # Encoder pre-net prenet_out = prenet(inputs, is_training=is_training) # (N, T, E/2) # Encoder CBHG ## Conv1D bank enc = conv1d_banks(prenet_out, K=hp.encoder_num_banks, is_training=is_training) # (N, T, K * E / 2) ### Max pooling enc = tf.layers.max_pooling1d(enc, 2, 1, padding="same") # (N, T, K * E / 2) ### Conv1D projections enc = conv1d(enc, hp.embed_size//2, 3, scope="conv1d_1") # (N, T, E/2) enc = normalize(enc, type=hp.norm_type, is_training=is_training, activation_fn=tf.nn.relu, scope="norm1") enc = conv1d(enc, hp.embed_size//2, 3, scope="conv1d_2") # (N, T, E/2) enc = normalize(enc, type=hp.norm_type, is_training=is_training, activation_fn=None, scope="norm2") enc += prenet_out # (N, T, E/2) # residual connections ### Highway Nets for i in range(hp.num_highwaynet_blocks): enc = highwaynet(enc, num_units=hp.embed_size//2, scope='highwaynet_{}'.format(i)) # (N, T, E/2) ### Bidirectional GRU memory = gru(enc, hp.embed_size//2, True) # (N, T, E) return memory
def main(): g = Graph(); print("Training Graph loaded") with g.graph.as_default(): # Load vocabulary char2idx, idx2char = load_vocab() # Training sv = tf.train.Supervisor(logdir=hp.logdir, save_model_secs=0) with sv.managed_session() as sess: for epoch in range(1, hp.num_epochs+1): if sv.should_stop(): break for step in tqdm(range(g.num_batch), total=g.num_batch, ncols=70, leave=False, unit='b'): sess.run(g.train_op) # Write checkpoint files at every epoch gs = sess.run(g.global_step) sv.saver.save(sess, hp.logdir + '/model_epoch_%02d_gs_%d' % (epoch, gs))
def __init__(self, mode="train"): ''' Args: is_train: Boolean. If True, backprop is executed. ''' if mode == "train": self.x, self.y, self.num_batch = get_batch_data( ) # (64, 50) int64, (64, 50) int64, 1636 else: # test self.x = tf.placeholder(tf.int64, [None, Hyperparams.maxlen]) # make embedding matrix for input characters pnyn2idx, _, hanzi2idx, _ = load_vocab() self.emb_x = tf.sg_emb(name='emb_x', voca_size=len(pnyn2idx), dim=Hyperparams.embed_dim) self.enc = self.x.sg_lookup(emb=self.emb_x) with tf.sg_context(size=5, act='relu', bn=True): for _ in range(20): dim = self.enc.get_shape().as_list()[-1] self.enc += self.enc.sg_conv1d( dim=dim) # (64, 50, 300) float32 # final fully convolutional layer for softmax self.logits = self.enc.sg_conv1d(size=1, dim=len(hanzi2idx), act='linear', bn=False) # (64, 50, 5072) float32 if mode == "train": self.ce = self.logits.sg_ce(target=self.y, mask=True) # (64, 50) float32 self.istarget = tf.not_equal(self.y, tf.zeros_like( self.y)).sg_float() # (64, 50) float32 self.reduced_loss = self.ce.sg_sum() / self.istarget.sg_sum( ) # () float32 tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
class Graph: # Load vocabulary char2idx, idx2char = load_vocab() def __init__(self, is_training=True): self.graph = tf.Graph() with self.graph.as_default(): if is_training: self.x, self.y, self.z, self.num_batch = get_batch() else: # Evaluation self.x = tf.placeholder(tf.int32, shape=(None, None)) self.y = tf.placeholder(tf.float32, shape=(None, None, hp.n_mels * hp.r)) self.decoder_inputs = shift_by_one(self.y) with tf.variable_scope("net"): # Encoder self.memory = encode(self.x, is_training=is_training) # (N, T, E) # Decoder self.outputs1 = decode1( self.decoder_inputs, self.memory, is_training=is_training) # (N, T', hp.n_mels*hp.r) self.outputs2 = decode2( self.outputs1, is_training=is_training) # (N, T', (1+hp.n_fft//2)*hp.r) if is_training: # Loss if hp.loss_type == "l1": # L1 loss self.loss1 = tf.abs(self.outputs1 - self.y) self.loss2 = tf.abs(self.outputs2 - self.z) else: # L2 loss self.loss1 = tf.squared_difference(self.outputs1, self.y) self.loss2 = tf.squared_difference(self.outputs2, self.z) # Target masking if hp.target_zeros_masking: self.loss1 *= tf.to_float(tf.not_equal(self.y, 0.)) self.loss2 *= tf.to_float(tf.not_equal(self.z, 0.)) self.mean_loss1 = tf.reduce_mean(self.loss1) self.mean_loss2 = tf.reduce_mean(self.loss2) self.mean_loss = self.mean_loss1 + self.mean_loss2 # Logging ## histograms self.expected1_h = tf.reduce_mean(tf.reduce_mean(self.y, -1), 0) self.got1_h = tf.reduce_mean(tf.reduce_mean(self.outputs1, -1), 0) self.expected2_h = tf.reduce_mean(tf.reduce_mean(self.z, -1), 0) self.got2_h = tf.reduce_mean(tf.reduce_mean(self.outputs2, -1), 0) ## images self.expected1_i = tf.expand_dims( tf.reduce_mean(self.y[:1], -1, keep_dims=True), 1) self.got1_i = tf.expand_dims( tf.reduce_mean(self.outputs1[:1], -1, keep_dims=True), 1) self.expected2_i = tf.expand_dims( tf.reduce_mean(self.z[:1], -1, keep_dims=True), 1) self.got2_i = tf.expand_dims( tf.reduce_mean(self.outputs2[:1], -1, keep_dims=True), 1) # Training Scheme self.global_step = tf.Variable(0, name='global_step', trainable=False) self.optimizer = tf.train.AdamOptimizer(learning_rate=hp.lr) self.train_op = self.optimizer.minimize( self.mean_loss, global_step=self.global_step) # Summmary tf.summary.scalar('mean_loss1', self.mean_loss1) tf.summary.scalar('mean_loss2', self.mean_loss2) tf.summary.scalar('mean_loss', self.mean_loss) tf.summary.histogram('expected_values1', self.expected1_h) tf.summary.histogram('gotten_values1', self.got1_h) tf.summary.histogram('expected_values2', self.expected2_h) tf.summary.histogram('gotten values2', self.got2_h) tf.summary.image("expected_values1", self.expected1_i * 255) tf.summary.image("gotten_values1", self.got1_i * 255) tf.summary.image("expected_values2", self.expected2_i * 255) tf.summary.image("gotten_values2", self.got2_i * 255) self.merged = tf.summary.merge_all()