def test_transformer_trainable_and_decodable(model_dict): args = make_arg(**model_dict) model, x, ilens, y, data, uttid_list = prepare(args) # decoding params recog_args = argparse.Namespace( maskctc_n_iterations=args.maskctc_n_iterations, maskctc_probability_threshold=args.maskctc_probability_threshold, ) # test training optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(x, ilens, y) optim.zero_grad() loss.backward() optim.step() # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) plot.plot_multi_head_attention(data, uttid_list, attn_dict, "", savefn=_savefn) # test decoding with torch.no_grad(): model.recognize(x[0, :ilens[0]].numpy(), recog_args, args.char_list)
def test_transformer_trainable_and_decodable(module, model_dict): args = make_arg(**model_dict) model, y_src, ilens, y_tgt, data = prepare(module, args) # test beam search trans_args = argparse.Namespace( beam_size=1, penalty=0.0, ctc_weight=0.0, maxlenratio=1.0, lm_weight=0, minlenratio=0, nbest=1, tgt_lang=False, ) if module == "pytorch": # test trainable optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(y_src, ilens, y_tgt) optim.zero_grad() loss.backward() optim.step() # test attention plot attn_dict = model.calculate_all_attentions(y_src[0:1], ilens[0:1], y_tgt[0:1]) from espnet.nets.pytorch_backend.transformer import plot plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") # test decodable with torch.no_grad(): nbest = model.translate([y_src[0, :ilens[0]].numpy()], trans_args, args.char_list) print(y_tgt[0]) print(nbest[0]["yseq"][1:-1]) else: raise NotImplementedError
def test_transformer_trainable_and_decodable(model_dict): args = make_arg(**model_dict) model, x, ilens, y, data = prepare(args) # decoding params recog_args = argparse.Namespace( maskctc_n_iterations=args.maskctc_n_iterations, maskctc_probability_threshold=args.maskctc_probability_threshold, ) # test training optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(x, ilens, y) optim.zero_grad() loss.backward() optim.step() # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) from espnet.nets.pytorch_backend.transformer import plot plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") # test decoding with torch.no_grad(): nbest = model.recognize_maskctc(x[0, :ilens[0]].numpy(), recog_args, args.char_list) print(y[0]) print(nbest[0]["yseq"][1:-1])
def test_transformer_trainable_and_decodable(module, model_dict): args = make_arg(**model_dict) model, x, ilens, y, data = prepare(module, args) # check for pure CTC and pure Attention if args.mtlalpha == 1: assert model.decoder is None elif args.mtlalpha == 0: assert model.ctc is None # test beam search recog_args = argparse.Namespace( beam_size=1, penalty=0.0, ctc_weight=0.0, maxlenratio=1.0, lm_weight=0, minlenratio=0, nbest=1, ) if module == "pytorch": # test trainable optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(x, ilens, y) optim.zero_grad() loss.backward() optim.step() # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) from espnet.nets.pytorch_backend.transformer import plot plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") # test decodable with torch.no_grad(): nbest = model.recognize(x[0, :ilens[0]].numpy(), recog_args) print(y[0]) print(nbest[0]["yseq"][1:-1]) else: # test trainable optim = chainer.optimizers.Adam(0.01) optim.setup(model) loss, loss_ctc, loss_att, acc = model(x, ilens, y) model.cleargrads() loss.backward() optim.update() # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) from espnet.nets.pytorch_backend.transformer import plot plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") # test decodable with chainer.no_backprop_mode(): nbest = model.recognize(x[0, :ilens[0]], recog_args) print(y[0]) print(nbest[0]["yseq"][1:-1])
def test_calculate_plot_attention(): from espnet.nets.pytorch_backend.transformer import plot train_args = make_train_args(report_cer=True) model, x, ilens, y, data = prepare("pytorch", train_args) attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test")
def test_calculate_plot_attention(): from espnet.nets.pytorch_backend.transformer import plot train_args = make_train_args(report_cer=True) model, feats, feats_len, labels, data, uttid_list = prepare(train_args) model.attention_plot_class attn_dict = model.calculate_all_attentions(feats[0:1], feats_len[0:1], labels[0:1]) plot.plot_multi_head_attention(data, uttid_list, attn_dict, "/tmp/espnet-test")
def test_transformer_synth(module): T = importlib.import_module('espnet.nets.{}_backend.e2e_asr_transformer'.format(module)) model, x, ilens, y, data = prepare(module) # test beam search recog_args = Namespace( beam_size=1, penalty=0.0, ctc_weight=0.0, maxlenratio=0, minlenratio=0, nbest=1 ) # test acc is almost 100% if module == "pytorch": optim = torch.optim.Adam(model.parameters(), 0.01) max_acc = 0 for i in range(40): loss = model(x, ilens, y) optim.zero_grad() loss.backward() optim.step() max_acc = max(model.acc, max_acc) assert max_acc > 0.8 # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) from espnet.nets.pytorch_backend.transformer import plot plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") with torch.no_grad(): nbest = model.recognize(x[0, :ilens[0]].numpy(), recog_args) print(y[0]) print(nbest[0]["yseq"][1:-1]) else: optim = chainer.optimizers.Adam(0.01) optim.setup(model) max_acc = 0 for i in range(40): loss, loss_ctc, loss_att, acc = model(x, ilens, y) model.cleargrads() loss.backward() optim.update() print(loss_att, acc) max_acc = max(acc.data, max_acc) assert max_acc > 0.8 # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) T.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") with chainer.no_backprop_mode(): nbest = model.recognize(x[0, :ilens[0]], recog_args) print(y[0]) print(nbest[0]["yseq"][1:-1])
def test_transformer_trainable_and_decodable(model_dict): args = make_arg(**model_dict) model, x, ilens, y, data, uttid_list = prepare(args) # check for pure CTC and pure Attention if args.mtlalpha == 1: assert model.decoder is None elif args.mtlalpha == 0: assert model.ctc is None # test beam search recog_args = argparse.Namespace( beam_size=1, penalty=0.0, ctc_weight=0.0, maxlenratio=1.0, lm_weight=0, minlenratio=0, nbest=1, ) # test trainable optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(x, ilens, y) optim.zero_grad() loss.backward() optim.step() # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) plot.plot_multi_head_attention(data, uttid_list, attn_dict, "", savefn=_savefn) # test CTC plot ctc_probs = model.calculate_all_ctc_probs(x[0:1], ilens[0:1], y[0:1]) if args.mtlalpha > 0: print(ctc_probs.shape) else: assert ctc_probs is None # test decodable with torch.no_grad(): nbest = model.recognize(x[0, :ilens[0]].numpy(), recog_args) print(y[0]) print(nbest[0]["yseq"][1:-1])
def test_transformer_trainable_and_decodable(model_dict): args = make_arg(**model_dict) model, x, ilens, y_tgt, y_src, data, uttid_list = prepare(args) # test beam search trans_args = argparse.Namespace( beam_size=1, penalty=0.0, ctc_weight=0.0, maxlenratio=1.0, lm_weight=0, minlenratio=0, nbest=1, tgt_lang=False, ) # test trainable optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(x, ilens, y_tgt, y_src) optim.zero_grad() loss.backward() optim.step() # test attention plot attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y_tgt[0:1], y_src[0:1]) plot.plot_multi_head_attention(data, uttid_list, attn_dict, "", savefn=_savefn) # test CTC plot ctc_probs = model.calculate_all_ctc_probs(x[0:1], ilens[0:1], y_tgt[0:1], y_src[0:1]) if args.asr_weight > 0 and args.mtlalpha > 0: print(ctc_probs.shape) else: assert ctc_probs is None # test decodable with torch.no_grad(): nbest = model.translate(x[0, :ilens[0]].numpy(), trans_args, args.char_list) print(y_tgt[0]) print(nbest[0]["yseq"][1:-1])
def test_sa_transducer_trainable_and_decodable(train_dic, recog_dic): from espnet.nets.pytorch_backend.transformer import plot train_args = make_train_args(**train_dic) recog_args = make_recog_args(**recog_dic) model, x, ilens, y, data = prepare('pytorch', train_args) optim = torch.optim.Adam(model.parameters(), 0.01) loss = model(x, ilens, y) optim.zero_grad() loss.backward() optim.step() attn_dict = model.calculate_all_attentions(x[0:1], ilens[0:1], y[0:1]) plot.plot_multi_head_attention(data, attn_dict, "/tmp/espnet-test") with torch.no_grad(): nbest = model.recognize(x[0, :ilens[0]].numpy(), recog_args) print(y[0]) print(nbest[0]["yseq"][1:-1])