def main():
g = Graph()
print("Training Graph loaded")
with g.graph.as_default():
# Load vocabulary
char2idx, idx2char = load_vocab()
# Training
sv = tf.train.Supervisor(logdir=hp.logdir, save_model_secs=0)
with sv.managed_session() as sess:
for epoch in range(1, hp.num_epochs + 1):
if sv.should_stop():
break
for step in tqdm(range(g.num_batch),
total=g.num_batch,
ncols=70,
leave=False,
unit='b'):
sess.run(g.train_op)
# Write checkpoint files at every epoch
gs = sess.run(g.global_step)
sv.saver.save(
sess, hp.logdir + '/model_epoch_%02d_gs_%d' % (epoch, gs))
def main():
g = ModelGraph(mode="test")
with tf.Session() as sess:
tf.sg_init(sess)
# restore parameters
saver = tf.train.Saver()
if Hyperparams.isqwerty:
save_path = "qwerty/asset/train/ckpt"
else:
save_path = "nine/asset/train/ckpt"
saver.restore(sess, tf.train.latest_checkpoint(save_path))
mname = open(save_path + "/checkpoint", 'r').read().split('"')[1]
nums, X, expected_list = load_test_data()
pnyn2idx, idx2pnyn, hanzi2idx, idx2hanzi = load_vocab()
with codecs.open('data/output_{}.txt'.format(mname), 'w',
'utf-8') as fout:
cum_score = 0
full_score = 0
for step in range(len(X) // 64 + 1):
n = nums[step * 64:(step + 1) * 64] #number batch
x = X[step * 64:(step + 1) * 64] # input batch
e = expected_list[step * 64:(step + 1) *
64] # batch of ground truth strings
# predict characters
logits = sess.run(g.logits, {g.x: x})
preds = np.squeeze(np.argmax(logits, -1))
for nn, xx, pp, ee in zip(n, x, preds, e): # sentence-wise
got = ''
for xxx, ppp in zip(xx, pp): # character-wise
if xxx == 0: break
if xxx == 1 or ppp == 1:
got += "*"
else:
got += idx2hanzi.get(ppp, "*")
got = got.replace("_", "") # Remove blanks
error = distance.levenshtein(ee, got)
score = len(ee) - error
cum_score += score
full_score += len(ee)
fout.write(u"{}\t{}\t{}\t{}\n".format(nn, ee, got, score))
fout.write(u"Total acc.: {}/{}={}\n".format(
cum_score, full_score, round(float(cum_score) / full_score,
2)))
def encode(inputs, is_training=True, scope="encoder", reuse=None):
'''
Args:
inputs: A 2d tensor with shape of [N, T], dtype of int32.
seqlens: A 1d tensor with shape of [N,], dtype of int32.
masks: A 3d tensor with shape of [N, T, 1], dtype of float32.
is_training: Whether or not the layer is in training mode.
scope: Optional scope for `variable_scope`
reuse: Boolean, whether to reuse the weights of a previous layer
by the same name.
Returns:
A collection of Hidden vectors, whose shape is (N, T, E).
'''
with tf.variable_scope(scope, reuse=reuse):
# Load vocabulary
char2idx, idx2char = load_vocab()
# Character Embedding
inputs = embed(inputs, len(char2idx), hp.embed_size) # (N, T, E)
# Encoder pre-net
prenet_out = prenet(inputs, is_training=is_training) # (N, T, E/2)
# Encoder CBHG
## Conv1D bank
enc = conv1d_banks(prenet_out, K=hp.encoder_num_banks, is_training=is_training) # (N, T, K * E / 2)
### Max pooling
enc = tf.layers.max_pooling1d(enc, 2, 1, padding="same") # (N, T, K * E / 2)
### Conv1D projections
enc = conv1d(enc, hp.embed_size//2, 3, scope="conv1d_1") # (N, T, E/2)
enc = normalize(enc, type=hp.norm_type, is_training=is_training,
activation_fn=tf.nn.relu, scope="norm1")
enc = conv1d(enc, hp.embed_size//2, 3, scope="conv1d_2") # (N, T, E/2)
enc = normalize(enc, type=hp.norm_type, is_training=is_training,
activation_fn=None, scope="norm2")
enc += prenet_out # (N, T, E/2) # residual connections
### Highway Nets
for i in range(hp.num_highwaynet_blocks):
enc = highwaynet(enc, num_units=hp.embed_size//2,
scope='highwaynet_{}'.format(i)) # (N, T, E/2)
### Bidirectional GRU
memory = gru(enc, hp.embed_size//2, True) # (N, T, E)
return memory
def main():
g = Graph(); print("Training Graph loaded")
with g.graph.as_default():
# Load vocabulary
char2idx, idx2char = load_vocab()
# Training
sv = tf.train.Supervisor(logdir=hp.logdir,
save_model_secs=0)
with sv.managed_session() as sess:
for epoch in range(1, hp.num_epochs+1):
if sv.should_stop(): break
for step in tqdm(range(g.num_batch), total=g.num_batch, ncols=70, leave=False, unit='b'):
sess.run(g.train_op)
# Write checkpoint files at every epoch
gs = sess.run(g.global_step)
sv.saver.save(sess, hp.logdir + '/model_epoch_%02d_gs_%d' % (epoch, gs))
def __init__(self, mode="train"):
'''
Args:
is_train: Boolean. If True, backprop is executed.
'''
if mode == "train":
self.x, self.y, self.num_batch = get_batch_data(
) # (64, 50) int64, (64, 50) int64, 1636
else: # test
self.x = tf.placeholder(tf.int64, [None, Hyperparams.maxlen])
# make embedding matrix for input characters
pnyn2idx, _, hanzi2idx, _ = load_vocab()
self.emb_x = tf.sg_emb(name='emb_x',
voca_size=len(pnyn2idx),
dim=Hyperparams.embed_dim)
self.enc = self.x.sg_lookup(emb=self.emb_x)
with tf.sg_context(size=5, act='relu', bn=True):
for _ in range(20):
dim = self.enc.get_shape().as_list()[-1]
self.enc += self.enc.sg_conv1d(
dim=dim) # (64, 50, 300) float32
# final fully convolutional layer for softmax
self.logits = self.enc.sg_conv1d(size=1,
dim=len(hanzi2idx),
act='linear',
bn=False) # (64, 50, 5072) float32
if mode == "train":
self.ce = self.logits.sg_ce(target=self.y,
mask=True) # (64, 50) float32
self.istarget = tf.not_equal(self.y, tf.zeros_like(
self.y)).sg_float() # (64, 50) float32
self.reduced_loss = self.ce.sg_sum() / self.istarget.sg_sum(
) # () float32
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
class Graph:
# Load vocabulary
char2idx, idx2char = load_vocab()
def __init__(self, is_training=True):
self.graph = tf.Graph()
with self.graph.as_default():
if is_training:
self.x, self.y, self.z, self.num_batch = get_batch()
else: # Evaluation
self.x = tf.placeholder(tf.int32, shape=(None, None))
self.y = tf.placeholder(tf.float32,
shape=(None, None, hp.n_mels * hp.r))
self.decoder_inputs = shift_by_one(self.y)
with tf.variable_scope("net"):
# Encoder
self.memory = encode(self.x,
is_training=is_training) # (N, T, E)
# Decoder
self.outputs1 = decode1(
self.decoder_inputs, self.memory,
is_training=is_training) # (N, T', hp.n_mels*hp.r)
self.outputs2 = decode2(
self.outputs1,
is_training=is_training) # (N, T', (1+hp.n_fft//2)*hp.r)
if is_training:
# Loss
if hp.loss_type == "l1": # L1 loss
self.loss1 = tf.abs(self.outputs1 - self.y)
self.loss2 = tf.abs(self.outputs2 - self.z)
else: # L2 loss
self.loss1 = tf.squared_difference(self.outputs1, self.y)
self.loss2 = tf.squared_difference(self.outputs2, self.z)
# Target masking
if hp.target_zeros_masking:
self.loss1 *= tf.to_float(tf.not_equal(self.y, 0.))
self.loss2 *= tf.to_float(tf.not_equal(self.z, 0.))
self.mean_loss1 = tf.reduce_mean(self.loss1)
self.mean_loss2 = tf.reduce_mean(self.loss2)
self.mean_loss = self.mean_loss1 + self.mean_loss2
# Logging
## histograms
self.expected1_h = tf.reduce_mean(tf.reduce_mean(self.y, -1),
0)
self.got1_h = tf.reduce_mean(tf.reduce_mean(self.outputs1, -1),
0)
self.expected2_h = tf.reduce_mean(tf.reduce_mean(self.z, -1),
0)
self.got2_h = tf.reduce_mean(tf.reduce_mean(self.outputs2, -1),
0)
## images
self.expected1_i = tf.expand_dims(
tf.reduce_mean(self.y[:1], -1, keep_dims=True), 1)
self.got1_i = tf.expand_dims(
tf.reduce_mean(self.outputs1[:1], -1, keep_dims=True), 1)
self.expected2_i = tf.expand_dims(
tf.reduce_mean(self.z[:1], -1, keep_dims=True), 1)
self.got2_i = tf.expand_dims(
tf.reduce_mean(self.outputs2[:1], -1, keep_dims=True), 1)
# Training Scheme
self.global_step = tf.Variable(0,
name='global_step',
trainable=False)
self.optimizer = tf.train.AdamOptimizer(learning_rate=hp.lr)
self.train_op = self.optimizer.minimize(
self.mean_loss, global_step=self.global_step)
# Summmary
tf.summary.scalar('mean_loss1', self.mean_loss1)
tf.summary.scalar('mean_loss2', self.mean_loss2)
tf.summary.scalar('mean_loss', self.mean_loss)
tf.summary.histogram('expected_values1', self.expected1_h)
tf.summary.histogram('gotten_values1', self.got1_h)
tf.summary.histogram('expected_values2', self.expected2_h)
tf.summary.histogram('gotten values2', self.got2_h)
tf.summary.image("expected_values1", self.expected1_i * 255)
tf.summary.image("gotten_values1", self.got1_i * 255)
tf.summary.image("expected_values2", self.expected2_i * 255)
tf.summary.image("gotten_values2", self.got2_i * 255)
self.merged = tf.summary.merge_all()