def dev(iter_dev, tfdata_dev, dataset_dev, sess, G_infer):
list_res = []
list_ref = []
list_length = []
length_rate = []
process = 0
sess.run(tfdata_dev.initializer)
while 1:
try:
np.random.seed(process)
pinyin_G_dev, text_G_dev = sess.run(iter_dev)
text_lens_G_dev = pinyin_lens_G_dev = get_batch_length(
pinyin_G_dev)
feature_dev, feature_lens_G_dev = int2vector(
pinyin_G_dev,
pinyin_lens_G_dev,
hidden_size=args.model.dim_input,
uprate=args.uprate)
feature_dev += np.random.randn(*feature_dev.shape) / args.noise
_, samples, len_samples = sess.run(G_infer.run_list,
feed_dict={
G_infer.list_pl[0]:
feature_dev,
G_infer.list_pl[1]:
feature_lens_G_dev
})
list_res.extend(list(samples))
list_ref.extend(list(text_G_dev))
list_length.extend(list(text_lens_G_dev))
length_rate.extend(list(len_samples / text_lens_G_dev))
process += len(text_G_dev)
# if process // 5000 == 0:
# print('processed {} samples'.format(process))
except tf.errors.OutOfRangeError:
break
print('dev len_res/len_ref : {:.2f} process {} samples.'.format(
np.mean(length_rate), process))
if args.uprate == 1.0:
acc = accuracy(list_res, list_ref, list_length)
print('dev accuracy: {:.2f}%'.format(acc * 100.0))
else:
cer_dist, cer_len = batch_cer(result=list_res, reference=list_ref)
cer = cer_dist / cer_len
print('dev cer: {:.2f}%'.format(cer * 100.0))
return pinyin_G_dev, text_G_dev
def decode(text_G_dev, iter_text_dev, sess, G_infer):
text_lens_G_dev = get_batch_length(text_G_dev)
feature_dev, feature_lens_G_dev = int2vector(text_G_dev, text_lens_G_dev, hidden_size=args.model.dim_input , uprate=args.uprate)
feature_dev += np.random.randn(*feature_dev.shape)/args.noise
logits, samples, len_samples = sess.run(G_infer.run_list,
feed_dict={G_infer.list_pl[0]:feature_dev,
G_infer.list_pl[1]:feature_lens_G_dev})
list_res_txt = array_idx2char(samples, args.idx2token, seperator=' ')
list_ref_txt = array_idx2char(text_G_dev, args.idx2token, seperator=' ')
for res, ref in zip(list_res_txt[:5], list_ref_txt):
# text_sample = args.idx2token[sample]
print('sampled: ', res)
print('label : ', ref)
def train():
dataset_train = PinyinDataSet(list_files=[args.dirs.text.data],
args=args,
_shuffle=True)
tfdata_train = tf.data.Dataset.from_generator(
dataset_train, (tf.int32, tf.int32),
(tf.TensorShape([None]), tf.TensorShape([None])))
iter_train = tfdata_train.cache().repeat().shuffle(10000).\
padded_batch(args.text_batch_size, ([args.max_label_len], [args.max_label_len])).prefetch(buffer_size=5).\
make_one_shot_iterator().get_next()
dataset_supervise = PinyinDataSet(list_files=[args.dirs.text.supervise],
args=args,
_shuffle=True)
tfdata_supervise = tf.data.Dataset.from_generator(
dataset_supervise, (tf.int32, tf.int32),
(tf.TensorShape([None]), tf.TensorShape([None])))
iter_supervise = tfdata_supervise.cache().repeat().shuffle(100).\
padded_batch(args.num_supervised, ([args.max_label_len], [args.max_label_len])).prefetch(buffer_size=5).\
make_one_shot_iterator().get_next()
dataset_dev = PinyinDataSet(list_files=[args.dirs.text.dev],
args=args,
_shuffle=False)
tfdata_dev = tf.data.Dataset.from_generator(
dataset_dev, (tf.int32, tf.int32),
(tf.TensorShape([None]), tf.TensorShape([None])))
tfdata_dev = tfdata_dev.cache().\
padded_batch(args.text_batch_size, ([args.max_label_len], [args.max_label_len])).prefetch(buffer_size=5).\
make_initializable_iterator()
iter_dev = tfdata_dev.get_next()
tensor_global_step = tf.Variable(0, dtype=tf.int32, trainable=False)
tensor_global_step0 = tf.Variable(0, dtype=tf.int32, trainable=False)
tensor_global_step1 = tf.Variable(0, dtype=tf.int32, trainable=False)
G = Generator(tensor_global_step, training=True, args=args)
G_infer = Generator(tensor_global_step, training=False, args=args)
vars_G = G.trainable_variables
D = args.Model_D(tensor_global_step1,
training=True,
name='discriminator',
args=args)
gan = Conditional_GAN([tensor_global_step0, tensor_global_step1],
G,
D,
batch=None,
unbatch=None,
name='text_gan',
args=args)
size_variables()
start_time = datetime.now()
saver_G = tf.train.Saver(vars_G, max_to_keep=1)
saver = tf.train.Saver(max_to_keep=15)
summary = Summary(str(args.dir_log))
config = tf.ConfigProto()
config.allow_soft_placement = True
config.gpu_options.allow_growth = True
config.log_device_placement = False
with tf.train.MonitoredTrainingSession(config=config) as sess:
if args.dirs.checkpoint_G:
saver_G.restore(sess, args.dirs.checkpoint_G)
print('revocer G from', args.dirs.checkpoint_G)
# np.random.seed(0)
pinyin_supervise, text_supervise = sess.run(iter_supervise)
text_len_supervise = pinyin_len_supervise = get_batch_length(
pinyin_supervise)
feature_supervise, feature_len_supervise = int2vector(
pinyin_supervise,
pinyin_len_supervise,
hidden_size=args.model.dim_input,
uprate=args.uprate)
feature_supervise += np.random.randn(
*feature_supervise.shape) / args.noise
# supervise
for _ in range(500):
sess.run(G.run_list,
feed_dict={
G.list_pl[0]: feature_supervise,
G.list_pl[1]: feature_len_supervise,
G.list_pl[2]: text_supervise,
G.list_pl[3]: text_len_supervise
})
batch_time = time()
global_step = 0
while global_step < 99999999:
# global_step, lr_G, lr_D = sess.run([tensor_global_step0, gan.learning_rate_G, gan.learning_rate_D])
global_step, lr_G = sess.run(
[tensor_global_step0, G.learning_rate])
pinyin_supervise, text_supervise = sess.run(iter_supervise)
pinyin_len_supervise = get_batch_length(pinyin_supervise)
feature_supervise, feature_len_supervise = int2vector(
pinyin_supervise,
pinyin_len_supervise,
hidden_size=args.model.dim_input,
uprate=args.uprate)
feature_supervise += np.random.randn(
*feature_supervise.shape) / args.noise
# supervise
# for _ in range(1):
# loss_G_supervise, _ = sess.run(G.run_list,
# feed_dict={G.list_pl[0]:feature_text_supervise,
# G.list_pl[1]:text_len_supervise,
# G.list_pl[2]:text_supervise,
# G.list_pl[3]:text_len_supervise})
# generator input
pinyin_G, text_G = sess.run(iter_train)
pinyin_lens_G = get_batch_length(pinyin_G)
feature_G, lens_G = int2vector(pinyin_G,
pinyin_lens_G,
hidden_size=args.model.dim_input,
uprate=args.uprate)
feature_G += np.random.randn(*feature_G.shape) / args.noise
loss_G, loss_G_supervise, _ = sess.run(gan.list_train_G,
feed_dict={
gan.list_G_pl[0]:
feature_G,
gan.list_G_pl[1]:
lens_G,
gan.list_G_pl[2]:
feature_supervise,
gan.list_G_pl[3]:
feature_len_supervise,
gan.list_G_pl[4]:
text_supervise,
gan.list_G_pl[5]:
pinyin_len_supervise
})
# loss_G = loss_G_supervise = 0
# discriminator input
# for _ in range(5):
# # np.random.seed(2)
# pinyin_G, text_G = sess.run(iter_train)
# pinyin_lens_G = get_batch_length(pinyin_G)
# feature_G, feature_lens_G = int2vector(pinyin_G, pinyin_lens_G, hidden_size=args.model.dim_input, uprate=args.uprate)
# feature_G += np.random.randn(*feature_G.shape)/args.noise
#
# pinyin_D, text_D = sess.run(iter_train)
# text_lens_D = get_batch_length(text_D)
# shape_text = text_D.shape
# loss_D, loss_D_res, loss_D_text, loss_gp, _ = sess.run(gan.list_train_D,
# feed_dict={gan.list_D_pl[0]:text_D,
# gan.list_D_pl[1]:text_lens_D,
# gan.list_G_pl[0]:feature_G,
# gan.list_G_pl[1]:feature_lens_G})
loss_D_res = loss_D_text = loss_gp = 0
shape_text = [0, 0, 0]
# loss_D_res = - loss_G
# loss_G = loss_G_supervise = 0.0
# loss_D = loss_D_text = loss_gp = 0.0
# train
# if global_step % 5 == 0:
# for _ in range(2):
# loss_supervise, shape_batch, _, _ = sess.run(G.list_run)
# loss_G_supervise = 0
used_time = time() - batch_time
batch_time = time()
if global_step % 10 == 0:
# print('loss_G: {:.2f} loss_G_supervise: {:.2f} loss_D_res: {:.2f} loss_D_text: {:.2f} step: {}'.format(
# loss_G, loss_G_supervise, loss_D_res, loss_D_text, global_step))
print(
'loss_G_supervise: {:.2f} loss res|real|gp: {:.2f}|{:.2f}|{:.2f}\tbatch: {}\tlr:{:.1e} {:.2f}s step: {}'
.format(loss_G_supervise, loss_D_res, loss_D_text, loss_gp,
shape_text, lr_G, used_time, global_step))
# summary.summary_scalar('loss_G', loss_G, global_step)
# summary.summary_scalar('loss_D', loss_D, global_step)
# summary.summary_scalar('lr_G', lr_G, global_step)
# summary.summary_scalar('lr_D', lr_D, global_step)
if global_step % args.save_step == args.save_step - 1:
saver.save(get_session(sess),
str(args.dir_checkpoint / 'model'),
global_step=global_step,
write_meta_graph=True)
print('saved model as',
str(args.dir_checkpoint) + '/model-' + str(global_step))
# if global_step % args.dev_step == args.dev_step - 1:
if global_step % args.dev_step == 1:
pinyin_G_dev, text_G_dev = dev(iter_dev, tfdata_dev,
dataset_dev, sess, G_infer)
# if global_step % args.decode_step == args.decode_step - 1:
if global_step % args.decode_step == args.decode_step - 1:
decode(pinyin_G_dev, text_G_dev, sess, G_infer)
logging.info('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))
def train():
print('reading data form ', args.dirs.train.tfdata)
dataReader_train = TFReader(args.dirs.train.tfdata, args=args)
batch_train = dataReader_train.fentch_batch_bucket()
dataReader_untrain = TFReader(args.dirs.untrain.tfdata, args=args)
batch_untrain = dataReader_untrain.fentch_batch(args.batch_size)
# batch_untrain = dataReader_untrain.fentch_batch_bucket()
args.dirs.untrain.tfdata = Path(args.dirs.untrain.tfdata)
args.data.untrain_size = TFData.read_tfdata_info(
args.dirs.untrain.tfdata)['size_dataset']
dataset_text = TextDataSet(list_files=[args.dirs.text.data],
args=args,
_shuffle=True)
tfdata_train = tf.data.Dataset.from_generator(dataset_text, (tf.int32),
(tf.TensorShape([None])))
iter_text = tfdata_train.cache().repeat().shuffle(1000).\
padded_batch(args.text_batch_size, ([args.max_label_len])).prefetch(buffer_size=5).\
make_one_shot_iterator().get_next()
feat, label = readTFRecord(args.dirs.dev.tfdata,
args,
_shuffle=False,
transform=True)
dataloader_dev = ASRDataLoader(args.dataset_dev,
args,
feat,
label,
batch_size=args.batch_size,
num_loops=1)
tensor_global_step = tf.train.get_or_create_global_step()
tensor_global_step0 = tf.Variable(0, dtype=tf.int32, trainable=False)
tensor_global_step1 = tf.Variable(0, dtype=tf.int32, trainable=False)
G = args.Model(tensor_global_step,
encoder=args.model.encoder.type,
decoder=args.model.decoder.type,
batch=batch_train,
training=True,
args=args)
G_infer = args.Model(tensor_global_step,
encoder=args.model.encoder.type,
decoder=args.model.decoder.type,
training=False,
args=args)
vars_G = G.trainable_variables
D = args.Model_D(tensor_global_step1,
training=True,
name='discriminator',
args=args)
gan = args.GAN([tensor_global_step0, tensor_global_step1],
G,
D,
batch=batch_train,
unbatch=batch_untrain,
name='GAN',
args=args)
size_variables()
start_time = datetime.now()
saver_G = tf.train.Saver(vars_G, max_to_keep=1)
saver = tf.train.Saver(max_to_keep=15)
summary = Summary(str(args.dir_log))
config = tf.ConfigProto()
config.allow_soft_placement = True
config.gpu_options.allow_growth = True
config.log_device_placement = False
with tf.train.MonitoredTrainingSession(config=config) as sess:
dataloader_dev.sess = sess
if args.dirs.checkpoint_G:
saver_G.restore(sess, args.dirs.checkpoint_G)
for i in range(2000):
ctc_loss, *_ = sess.run(G.list_run)
if i % 400:
print('ctc_loss: {:.2f}'.format(ctc_loss))
batch_time = time()
num_processed = 0
num_processed_unbatch = 0
progress = 0
while progress < args.num_epochs:
global_step, lr_G, lr_D = sess.run([
tensor_global_step0, gan.learning_rate_G, gan.learning_rate_D
])
# untrain
text = sess.run(iter_text)
text_lens = get_batch_length(text)
shape_text = text.shape
# loss_D, loss_D_res, loss_D_text, loss_gp, _ = sess.run(gan.list_train_D,
# feed_dict={gan.list_pl[0]:text,
# gan.list_pl[1]:text_lens})
loss_D = loss_D_res = loss_D_text = loss_gp = 0
(loss_G, loss_G_supervise, _), (shape_feature, shape_unfeature) = \
sess.run([gan.list_train_G, gan.list_feature_shape])
num_processed += shape_feature[0]
num_processed_unbatch += shape_unfeature[0]
used_time = time() - batch_time
batch_time = time()
progress = num_processed / args.data.train_size
progress_unbatch = num_processed_unbatch / args.data.untrain_size
if global_step % 20 == 0:
print(
'loss_supervise: {:.2f}, loss res|real|gp: {:.2f}|{:.2f}|{:.2f}\tbatch: {}|{}|{}\tlr:{:.1e}|{:.1e} {:.2f}s {:.1f}%|{:.1f}% step: {}'
.format(loss_G_supervise, loss_D_res, loss_D_text, loss_gp,
shape_feature, shape_unfeature, shape_text, lr_G,
lr_D, used_time, progress * 100.0,
progress_unbatch * 100.0, global_step))
# summary.summary_scalar('loss_G', loss_G, global_step)
# summary.summary_scalar('loss_D', loss_D, global_step)
# summary.summary_scalar('lr_G', lr_G, global_step)
# summary.summary_scalar('lr_D', lr_D, global_step)
if global_step % args.save_step == args.save_step - 1:
saver.save(get_session(sess),
str(args.dir_checkpoint / 'model'),
global_step=global_step,
write_meta_graph=True)
# if global_step % args.dev_step == args.dev_step - 1:
if global_step % args.dev_step == 0:
cer, wer = dev(step=global_step,
dataloader=dataloader_dev,
model=G_infer,
sess=sess,
unit=args.data.unit,
idx2token=args.idx2token,
token2idx=args.token2idx)
# summary.summary_scalar('dev_cer', cer, global_step)
# summary.summary_scalar('dev_wer', wer, global_step)
if global_step % args.decode_step == args.decode_step - 1:
decode_test(step=global_step,
sample=args.dataset_test[10],
model=G_infer,
sess=sess,
unit=args.data.unit,
idx2token=args.idx2token,
token2idx=args.token2idx)
logging.info('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))
def train_gan():
print('reading data form ', args.dirs.train.tfdata)
dataReader_train = TFDataReader(args.dirs.train.tfdata, args=args)
batch_train = dataReader_train.fentch_multi_batch_bucket()
dataReader_untrain = TFDataReader(args.dirs.untrain.tfdata, args=args)
batch_untrain = dataReader_untrain.fentch_multi_batch(args.batch_size)
args.dirs.untrain.tfdata = Path(args.dirs.untrain.tfdata)
args.data.untrain_size = TFDataReader.read_tfdata_info(
args.dirs.untrain.tfdata)['size_dataset']
dataset_text = TextDataSet(list_files=[args.dirs.text.data],
args=args,
_shuffle=True)
tfdata_train = tf.data.Dataset.from_generator(dataset_text, (tf.int32),
(tf.TensorShape([None])))
iter_text = tfdata_train.cache().repeat().shuffle(1000).\
padded_batch(args.text_batch_size, ([args.max_label_len])).prefetch(buffer_size=5).\
make_one_shot_iterator().get_next()
dataReader_dev = TFDataReader(args.dirs.dev.tfdata,
args=args,
_shuffle=False,
transform=True)
dataloader_dev = ASR_Multi_DataLoader(args.dataset_dev,
args,
dataReader_dev.feat,
dataReader_dev.phone,
dataReader_dev.label,
batch_size=args.batch_size,
num_loops=1)
tensor_global_step = tf.train.get_or_create_global_step()
tensor_global_step0 = tf.Variable(0, dtype=tf.int32, trainable=False)
tensor_global_step1 = tf.Variable(0, dtype=tf.int32, trainable=False)
G = args.Model(tensor_global_step,
encoder=args.model.encoder.type,
encoder2=args.model.encoder2.type,
decoder=args.model.decoder.type,
batch=batch_train,
training=True,
args=args)
G_infer = args.Model(tensor_global_step,
encoder=args.model.encoder.type,
encoder2=args.model.encoder2.type,
decoder=args.model.decoder.type,
training=False,
args=args)
vars_ASR = G.trainable_variables()
# vars_G_ocd = G.trainable_variables('Ectc_Docd/ocd_decoder')
D = args.Model_D(tensor_global_step1,
training=True,
name='discriminator',
args=args)
gan = args.GAN([tensor_global_step0, tensor_global_step1],
G,
D,
batch=batch_train,
unbatch=batch_untrain,
name='GAN',
args=args)
size_variables()
start_time = datetime.now()
saver_ASR = tf.train.Saver(vars_ASR, max_to_keep=10)
saver = tf.train.Saver(max_to_keep=15)
summary = Summary(str(args.dir_log))
config = tf.ConfigProto()
config.allow_soft_placement = True
config.gpu_options.allow_growth = True
config.log_device_placement = False
with tf.train.MonitoredTrainingSession(config=config) as sess:
dataloader_dev.sess = sess
if args.dirs.checkpoint_G:
saver_ASR.restore(sess, args.dirs.checkpoint_G)
batch_time = time()
num_processed = 0
num_processed_unbatch = 0
progress = 0
while progress < args.num_epochs:
# semi_supervise
global_step, lr_G, lr_D = sess.run([
tensor_global_step0, gan.learning_rate_G, gan.learning_rate_D
])
for _ in range(3):
text = sess.run(iter_text)
text_lens = get_batch_length(text)
shape_text = text.shape
loss_D, loss_D_res, loss_D_text, loss_gp, _ = sess.run(
gan.list_train_D,
feed_dict={
gan.list_pl[0]: text,
gan.list_pl[1]: text_lens
})
# loss_D=loss_D_res=loss_D_text=loss_gp=0
(loss_G, ctc_loss, ce_loss, _), (shape_batch, shape_unbatch) = \
sess.run([gan.list_train_G, gan.list_feature_shape])
num_processed += shape_batch[0]
# num_processed_unbatch += shape_unbatch[0]
used_time = time() - batch_time
batch_time = time()
progress = num_processed / args.data.train_size
progress_unbatch = num_processed_unbatch / args.data.untrain_size
if global_step % 40 == 0:
print('ctc|ce loss: {:.2f}|{:.2f}, loss res|real|gp: {:.2f}|{:.2f}|{:.2f}\t{}|{}\tlr:{:.1e}|{:.1e} {:.2f}s {:.3f}% step: {}'.format(
np.mean(ctc_loss), np.mean(ce_loss), loss_D_res, loss_D_text, loss_gp, shape_batch, \
shape_unbatch, lr_G, lr_D, used_time, progress*100, global_step))
summary.summary_scalar('ctc_loss', np.mean(ctc_loss),
global_step)
summary.summary_scalar('ce_loss', np.mean(ce_loss),
global_step)
if global_step % args.save_step == args.save_step - 1:
saver_ASR.save(get_session(sess),
str(args.dir_checkpoint / 'model_G'),
global_step=global_step,
write_meta_graph=True)
print(
'saved G in',
str(args.dir_checkpoint) + '/model_G-' + str(global_step))
# saver_G_en.save(get_session(sess), str(args.dir_checkpoint/'model_G_en'), global_step=global_step, write_meta_graph=True)
# print('saved model in', str(args.dir_checkpoint)+'/model_G_en-'+str(global_step))
if global_step % args.dev_step == args.dev_step - 1:
# if True:
per, cer, wer = dev(step=global_step,
dataloader=dataloader_dev,
model=G_infer,
sess=sess,
unit=args.data.unit,
args=args)
summary.summary_scalar('dev_per', per, global_step)
summary.summary_scalar('dev_cer', cer, global_step)
summary.summary_scalar('dev_wer', wer, global_step)
if global_step % args.decode_step == args.decode_step - 1:
# if True:
decode_test(step=global_step,
sample=args.dataset_test.uttid2sample(
args.sample_uttid),
model=G_infer,
sess=sess,
unit=args.data.unit,
args=args)
logging.info('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))