def Decode(save_file):
dataset_dev = ASR_align_ArkDataSet(scp_file=args.dirs.dev.scp,
trans_file=args.dirs.dev.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
feature_dev = TFData(dataset=dataset_dev,
dir_save=args.dirs.dev.tfdata,
args=args).read()
feature_dev = feature_dev.padded_batch(args.batch_size,
((), [None, args.dim_input]))
encoder = Encoder(args)
decoder = Decoder(args)
D = CLM(args)
encoder.summary()
decoder.summary()
D.summary()
ckpt_G = tf.train.Checkpoint(encoder=encoder, decoder=decoder)
_ckpt_manager = tf.train.CheckpointManager(ckpt_G,
args.dirs.checkpoint,
max_to_keep=1)
ckpt_G.restore(_ckpt_manager.latest_checkpoint)
print('checkpoint {} restored!!'.format(_ckpt_manager.latest_checkpoint))
cer = evaluate(feature_dev, dataset_dev, args.data.dev_size, encoder,
decoder)
print('PER:{:.3f}'.format(cer))
def Decode(save_file):
dataset_dev = ASR_align_ArkDataSet(scp_file=args.dirs.dev.scp,
trans_file=args.dirs.dev.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
feature_dev = TFData(dataset=dataset_dev,
dir_save=args.dirs.dev.tfdata,
args=args).read()
feature_dev = feature_dev.padded_batch(args.batch_size,
((), [None, args.dim_input]))
model = Model(args)
model.summary()
optimizer = tf.keras.optimizers.Adam(1e-4)
ckpt = tf.train.Checkpoint(model=model, optimizer=optimizer)
_ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dirs.checkpoint,
max_to_keep=1)
ckpt.restore(_ckpt_manager.latest_checkpoint)
print('checkpoint {} restored!!'.format(_ckpt_manager.latest_checkpoint))
cer = evaluate(feature_dev, dataset_dev, args.data.dev_size, model)
print('PER:{:.3f}'.format(cer))
def main():
dataset_train = ASR_align_ArkDataSet(scp_file=args.dirs.train.scp,
trans_file=args.dirs.train.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=True,
transform=False)
dataset_dev = ASR_align_ArkDataSet(scp_file=args.dirs.dev.scp,
trans_file=args.dirs.dev.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
dataset_untrain = ASR_align_ArkDataSet(scp_file=args.dirs.untrain.scp,
trans_file=None,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=True,
transform=False)
feature_train = TFData(dataset=dataset_train,
dir_save=args.dirs.train.tfdata,
args=args)
feature_untrain = TFData(dataset=dataset_untrain,
dir_save=args.dirs.untrain.tfdata,
args=args)
# feature_train_supervise = TFData(dataset=dataset_train_supervise,
# dir_save=args.dirs.train_supervise.tfdata,
# args=args)
feature_dev = TFData(dataset=dataset_dev,
dir_save=args.dirs.dev.tfdata,
args=args)
# feature_train.split_save(capacity=100000)
# feature_dev.split_save(capacity=100000)
feature_untrain.split_save(capacity=100000)
def Train():
with tf.device("/cpu:0"):
dataset_train = ASR_align_ArkDataSet(scp_file=args.dirs.train.scp,
trans_file=args.dirs.train.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
dataset_dev = ASR_align_ArkDataSet(scp_file=args.dirs.dev.scp,
trans_file=args.dirs.dev.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
# wav data
feature_train = TFData(dataset=dataset_train,
dir_save=args.dirs.train.tfdata,
args=args).read()
feature_dev = TFData(dataset=dataset_dev,
dir_save=args.dirs.dev.tfdata,
args=args).read()
bucket = tf.data.experimental.bucket_by_sequence_length(
element_length_func=lambda uttid, x: tf.shape(x)[0],
bucket_boundaries=args.list_bucket_boundaries,
bucket_batch_sizes=args.list_batch_size,
padded_shapes=((), [None, args.dim_input]))
iter_feature_train = iter(
feature_train.repeat().shuffle(10).apply(bucket).prefetch(
buffer_size=5))
# iter_feature_train = iter(feature_train.repeat().shuffle(500).padded_batch(args.batch_size,
# ((), [None, args.dim_input])).prefetch(buffer_size=5))
# feature_dev = feature_dev.apply(bucket).prefetch(buffer_size=5)
feature_dev = feature_dev.padded_batch(args.batch_size,
((), [None, args.dim_input]))
# create model paremeters
encoder = Encoder(args)
decoder = Decoder(args)
encoder.summary()
decoder.summary()
# lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
# args.opti.lr,
# decay_steps=args.opti.decay_steps,
# decay_rate=0.5,
# staircase=True)
optimizer = tf.keras.optimizers.Adam(args.opti.lr, beta_1=0.9, beta_2=0.98)
writer = tf.summary.create_file_writer(str(args.dir_log))
ckpt = tf.train.Checkpoint(encoder=encoder, decoder=decoder)
ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dir_checkpoint,
max_to_keep=20)
step = 0
# if a checkpoint exists, restore the latest checkpoint.
if args.dirs.checkpoint:
_ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dirs.checkpoint,
max_to_keep=1)
ckpt.restore(_ckpt_manager.latest_checkpoint)
print('checkpoint {} restored!!'.format(
_ckpt_manager.latest_checkpoint))
step = int(_ckpt_manager.latest_checkpoint.split('-')[-1])
start_time = datetime.now()
num_processed = 0
while step < 99999999:
start = time()
uttids, x = next(iter_feature_train)
trans = dataset_train.get_attrs('trans', uttids.numpy())
loss_supervise = train_CTC_supervised(x, trans, encoder, decoder,
optimizer)
num_processed += len(x)
progress = num_processed / args.data.train_size
if step % 10 == 0:
print('loss: {:.3f}\tbatch: {}\tused: {:.3f}\t {:.3f}% step: {}'.
format(loss_supervise, x.shape,
time() - start, progress * 100, step))
with writer.as_default():
tf.summary.scalar("costs/loss_supervise",
loss_supervise,
step=step)
if step % args.dev_step == 0:
cer = evaluate(feature_dev, dataset_dev, args.data.dev_size,
encoder, decoder)
with writer.as_default():
tf.summary.scalar("performance/cer", cer, step=step)
if step % args.decode_step == 0:
monitor(dataset_dev[0], encoder, decoder)
if step % args.save_step == 0:
save_path = ckpt_manager.save(step)
print('save model {}'.format(save_path))
step += 1
print('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))
def Train():
with tf.device("/cpu:0"):
dataset_train = ASR_align_ArkDataSet(scp_file=args.dirs.train.scp,
trans_file=args.dirs.train.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
dataset_dev = ASR_align_ArkDataSet(scp_file=args.dirs.dev.scp,
trans_file=args.dirs.dev.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
# wav data
feature_train = TFData(dataset=dataset_train,
dir_save=args.dirs.train.tfdata,
args=args).read()
feature_dev = TFData(dataset=dataset_dev,
dir_save=args.dirs.dev.tfdata,
args=args).read()
bucket = tf.data.experimental.bucket_by_sequence_length(
element_length_func=lambda uttid, x: tf.shape(x)[0],
bucket_boundaries=args.list_bucket_boundaries,
bucket_batch_sizes=args.list_batch_size,
padded_shapes=((), [None, args.dim_input]))
iter_feature_train = iter(
feature_train.repeat().shuffle(500).apply(bucket).prefetch(
buffer_size=5))
# iter_feature_train = iter(feature_train.repeat().shuffle(500).padded_batch(args.batch_size,
# ((), [None, args.dim_input])).prefetch(buffer_size=5))
feature_dev = feature_dev.padded_batch(args.batch_size,
((), [None, args.dim_input]))
stratedy = tf.distribute.MirroredStrategy(
devices=["device:GPU:0", "device:GPU:1"])
with stratedy.scope():
# create model paremeters
model = conv_lstm(args)
model.summary()
optimizer = tf.keras.optimizers.Adam(args.opti.lr,
beta_1=0.5,
beta_2=0.9)
writer = tf.summary.create_file_writer(str(args.dir_log))
ckpt = tf.train.Checkpoint(model=model, optimizer=optimizer)
ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dir_checkpoint,
max_to_keep=20)
step = 0
# if a checkpoint exists, restore the latest checkpoint.
if args.dirs.checkpoint:
_ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dirs.checkpoint,
max_to_keep=1)
ckpt.restore(_ckpt_manager.latest_checkpoint)
print('checkpoint {} restored!!'.format(
_ckpt_manager.latest_checkpoint))
step = int(_ckpt_manager.latest_checkpoint.split('-')[-1])
start_time = datetime.now()
num_processed = 0
while step < 99999999:
start = time()
@tf.function(experimental_relax_shapes=True)
def _train():
for _ in tf.range(1000):
stratedy.experimental_run_v2(train_CTC_supervised,
args=(next(iter_feature_train),
dataset_train, model,
optimizer))
# loss_supervise = tf.reduce_mean(res._values)
# loss_supervise = train_CTC_supervised(x, trans, model, optimizer)
# num_processed += len(x)
# progress = num_processed / args.data.train_size
# if step % 10 == 0:
# print('loss: {:.3f}\tused: {:.3f}\t step: {}'.format(
# loss_supervise, time()-start, step))
# with writer.as_default():
# tf.summary.scalar("costs/loss_supervise", loss_supervise, step=step)
# if step % args.dev_step == 0:
# cer = evaluate(feature_dev, dataset_dev, args.data.dev_size, model)
# with writer.as_default():
# tf.summary.scalar("performance/cer", cer, step=step)
# if step % args.decode_step == 0:
# monitor(dataset_dev[0], model)
# if step % args.save_step == 0:
# save_path = ckpt_manager.save(step)
# print('save model {}'.format(save_path))
#
# step += 1
print('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))
def Train():
args.data.untrain_size = TFData.read_tfdata_info(
args.dirs.untrain.tfdata)['size_dataset']
with tf.device("/cpu:0"):
dataset_train = ASR_align_ArkDataSet(scp_file=args.dirs.train.scp,
trans_file=args.dirs.train.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
dataset_untrain = ASR_align_ArkDataSet(scp_file=args.dirs.untrain.scp,
trans_file=None,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
dataset_dev = ASR_align_ArkDataSet(scp_file=args.dirs.dev.scp,
trans_file=args.dirs.dev.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=False)
# wav data
feature_train = TFData(dataset=dataset_train,
dir_save=args.dirs.train.tfdata,
args=args).read()
feature_unsupervise = TFData(dataset=dataset_untrain,
dir_save=args.dirs.untrain.tfdata,
args=args).read()
feature_dev = TFData(dataset=dataset_dev,
dir_save=args.dirs.dev.tfdata,
args=args).read()
bucket = tf.data.experimental.bucket_by_sequence_length(
element_length_func=lambda uttid, x: tf.shape(x)[0],
bucket_boundaries=args.list_bucket_boundaries,
bucket_batch_sizes=args.list_batch_size,
padded_shapes=((), [None, args.dim_input]))
iter_feature_train = iter(
feature_train.repeat().shuffle(100).apply(bucket).prefetch(
buffer_size=5))
# iter_feature_unsupervise = iter(feature_unsupervise.repeat().shuffle(100).apply(bucket).prefetch(buffer_size=5))
# iter_feature_train = iter(feature_train.repeat().shuffle(100).padded_batch(args.batch_size,
# ((), [None, args.dim_input])).prefetch(buffer_size=5))
iter_feature_unsupervise = iter(
feature_unsupervise.repeat().shuffle(100).padded_batch(
args.batch_size,
((), [None, args.dim_input])).prefetch(buffer_size=5))
# feature_dev = feature_dev.apply(bucket).prefetch(buffer_size=5)
feature_dev = feature_dev.padded_batch(args.batch_size,
((), [None, args.dim_input]))
dataset_text = TextDataSet(list_files=[args.dirs.lm.data],
args=args,
_shuffle=True)
tfdata_train = tf.data.Dataset.from_generator(dataset_text, (tf.int32),
(tf.TensorShape([None])))
iter_text = iter(tfdata_train.cache().repeat().shuffle(1000).map(
lambda x: x[:args.model.D.max_label_len]).padded_batch(
args.text_batch_size,
([args.model.D.max_label_len])).prefetch(buffer_size=5))
# create model paremeters
encoder = Encoder(args)
decoder = Decoder(args)
D = CLM(args)
encoder.summary()
decoder.summary()
D.summary()
optimizer = tf.keras.optimizers.Adam(0.0001, beta_1=0.5, beta_2=0.9)
optimizer_D = tf.keras.optimizers.Adam(0.0001, beta_1=0.5, beta_2=0.9)
writer = tf.summary.create_file_writer(str(args.dir_log))
ckpt_G = tf.train.Checkpoint(encoder=encoder, decoder=decoder)
ckpt_manager = tf.train.CheckpointManager(ckpt_G,
args.dir_checkpoint,
max_to_keep=20)
step = 0
if args.dirs.checkpoint_G:
_ckpt_manager = tf.train.CheckpointManager(ckpt_G,
args.dirs.checkpoint_G,
max_to_keep=1)
ckpt_G.restore(_ckpt_manager.latest_checkpoint)
print('checkpoint_G {} restored!!'.format(
_ckpt_manager.latest_checkpoint))
# cer = evaluate(feature_dev, dataset_dev, args.data.dev_size, encoder, decoder)
# with writer.as_default():
# tf.summary.scalar("performance/cer", cer, step=step)
start_time = datetime.now()
num_processed = 0
while step < 99999999:
start = time()
# supervise training
uttids, x = next(iter_feature_train)
trans = dataset_train.get_attrs('trans', uttids.numpy())
loss_supervise = train_CTC_supervised(x, trans, encoder, decoder,
optimizer)
# unsupervise training
text = next(iter_text)
_, un_x = next(iter_feature_unsupervise)
# loss_G = train_G(un_x, encoder, decoder, D, optimizer, args.model.D.max_label_len)
loss_G = train_G(un_x, encoder, decoder, D, optimizer,
args.model.D.max_label_len)
loss_D = train_D(un_x, text, encoder, decoder, D, optimizer_D,
args.lambda_gp, args.model.D.max_label_len)
num_processed += len(un_x)
progress = num_processed / args.data.untrain_size
if step % 10 == 0:
print(
'loss_supervise: {:.3f}\tloss_G: {:.3f}\tloss_D: {:.3f}\tbatch: {}\tused: {:.3f}\t {:.3f}% step: {}'
.format(loss_supervise, loss_G, loss_D, un_x.shape,
time() - start, progress * 100, step))
with writer.as_default():
tf.summary.scalar("costs/loss_supervise",
loss_supervise,
step=step)
if step % args.dev_step == args.dev_step - 1:
cer = evaluate(feature_dev, dataset_dev, args.data.dev_size,
encoder, decoder)
with writer.as_default():
tf.summary.scalar("performance/cer", cer, step=step)
if step % args.decode_step == 0:
monitor(dataset_dev[0], encoder, decoder)
if step % args.save_step == 0:
save_path = ckpt_manager.save(step)
print('save model {}'.format(save_path))
step += 1
print('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))
def Train():
with tf.device("/cpu:0"):
dataset_train = ASR_align_ArkDataSet(scp_file=args.dirs.train.scp,
trans_file=args.dirs.train.trans,
align_file=None,
feat_len_file=None,
args=args,
_shuffle=False,
transform=True)
feature_train = TFData(dataset=dataset_train,
dir_save=args.dirs.train.tfdata,
args=args).read(_shuffle=True, transform=True)
bucket = tf.data.experimental.bucket_by_sequence_length(
element_length_func=lambda uttid, x: tf.shape(x)[0],
bucket_boundaries=args.list_bucket_boundaries,
bucket_batch_sizes=args.list_batch_size,
padded_shapes=((), [None, args.dim_input]))
iter_feature_train = iter(
feature_train.repeat().shuffle(10).apply(bucket).prefetch(
buffer_size=5))
# create model paremeters
model = Transformer(args)
model.summary()
# learning_rate = CustomSchedule(args.model.G.d_model)
# lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
# 0.0001,
# decay_steps=10000,
# decay_rate=0.5,
# staircase=True)
optimizer = tf.keras.optimizers.Adam(0.000005,
beta_1=0.5,
beta_2=0.9,
epsilon=1e-9)
# optimizer = tf.keras.optimizers.SGD(0.1)
ckpt = tf.train.Checkpoint(model=model)
ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dir_checkpoint,
max_to_keep=20)
step = 0
# if a checkpoint exists, restore the latest checkpoint.
if args.dirs.checkpoint:
_ckpt_manager = tf.train.CheckpointManager(ckpt,
args.dirs.checkpoint,
max_to_keep=1)
ckpt.restore(_ckpt_manager.latest_checkpoint)
print('checkpoint {} restored!!'.format(
_ckpt_manager.latest_checkpoint))
step = int(_ckpt_manager.latest_checkpoint.split('-')[-1])
start_time = datetime.now()
num_processed = 0
# uttids, x = next(iter_feature_train)
# trans_sos = dataset_train.get_attrs('trans_sos', uttids.numpy())
# trans_eos = dataset_train.get_attrs('trans_eos', uttids.numpy())
while step < 99999999:
start = time()
uttids, x = next(iter_feature_train)
trans_sos = dataset_train.get_attrs('trans_sos', uttids.numpy())
trans_eos = dataset_train.get_attrs('trans_eos', uttids.numpy())
loss_supervise = train_step(x, trans_sos, trans_eos, model, optimizer)
num_processed += len(x)
progress = num_processed / args.data.train_size
if step % 20 == 0:
print('loss: {:.3f}\tbatch: {}\tused: {:.3f}\t {:.3f}% step: {}'.
format(loss_supervise, x.shape,
time() - start, progress * 100, step))
# if step % args.save_step == 0:
# save_path = ckpt_manager.save(step)
# print('save model {}'.format(save_path))
step += 1
print('training duration: {:.2f}h'.format(
(datetime.now() - start_time).total_seconds() / 3600))