def test_sortagrad_trainable_with_batch_frames(module, num_encs):
args = make_arg(num_encs=num_encs, sortagrad=1)
idim = 2
odim = 2
dummy_json = make_dummy_json(4, [2, 3], [2, 3],
idim=idim,
odim=odim,
num_inputs=num_encs)
import espnet.nets.pytorch_backend.e2e_asr_mulenc as m
batch_frames_in = 50
batch_frames_out = 50
batchset = make_batchset(
dummy_json,
batch_frames_in=batch_frames_in,
batch_frames_out=batch_frames_out,
shortest_first=True,
)
for batch in batchset:
i = 0
o = 0
for uttid, info in batch:
i += int(info["input"][0]["shape"][0]) # based on the first input
o += int(info["output"][0]["shape"][0])
assert i <= batch_frames_in
assert o <= batch_frames_out
model = m.E2E([2 for _ in range(num_encs)], 2, args)
for batch in batchset:
loss = model(
*convert_batch(batch, module, idim=2, odim=2, num_inputs=num_encs))
loss.backward() # trainable
with torch.no_grad():
in_data = [np.random.randn(100, 2) for _ in range(num_encs)]
model.recognize(in_data, args, args.char_list)
def test_sortagrad_trainable_with_batch_bins(module, num_encs):
args = make_arg(num_encs=num_encs, sortagrad=1)
idim = 20
odim = 5
dummy_json = make_dummy_json(4, [10, 20], [10, 20],
idim=idim,
odim=odim,
num_inputs=num_encs)
import espnet.nets.pytorch_backend.e2e_asr_mulenc as m
batch_elems = 2000
batchset = make_batchset(dummy_json,
batch_bins=batch_elems,
shortest_first=True)
for batch in batchset:
n = 0
for uttid, info in batch:
ilen = int(
info['input'][0]['shape'][0]) # based on the first input
olen = int(info['output'][0]['shape'][0])
n += ilen * idim + olen * odim
assert olen < batch_elems
model = m.E2E([20 for _ in range(num_encs)], 5, args)
for batch in batchset:
loss = model(*convert_batch(
batch, module, idim=20, odim=5, num_inputs=num_encs))
loss.backward() # trainable
with torch.no_grad(), chainer.no_backprop_mode():
in_data = [np.random.randn(100, 20) for _ in range(num_encs)]
model.recognize(in_data, args, args.char_list)
def test_gradient_noise_injection(module, num_encs):
args = make_arg(num_encs=num_encs, grad_noise=True)
args_org = make_arg(num_encs=num_encs)
dummy_json = make_dummy_json(num_encs, [10, 20], [10, 20], idim=20, odim=5, num_inputs=num_encs)
import espnet.nets.pytorch_backend.e2e_asr_mulenc as m
batchset = make_batchset(dummy_json, 2, 2 ** 10, 2 ** 10, shortest_first=True)
model = m.E2E([20 for _ in range(num_encs)], 5, args)
model_org = m.E2E([20 for _ in range(num_encs)], 5, args_org)
for batch in batchset:
loss = model(*convert_batch(batch, module, idim=20, odim=5, num_inputs=num_encs))
loss_org = model_org(*convert_batch(batch, module, idim=20, odim=5, num_inputs=num_encs))
loss.backward()
grad = [param.grad for param in model.parameters()][10]
loss_org.backward()
grad_org = [param.grad for param in model_org.parameters()][10]
assert grad[0] != grad_org[0]
def test_gpu_trainable(module, num_encs):
m = importlib.import_module(module)
args = make_arg(num_encs=num_encs)
model = m.E2E([2 for _ in range(num_encs)], 2, args)
if "pytorch" in module:
batch = prepare_inputs("pytorch", num_encs, is_cuda=True)
model.cuda()
loss = model(*batch)
loss.backward() # trainable
def test_calculate_all_attentions(module, num_encs, atype):
m = importlib.import_module(module)
args = make_arg(num_encs=num_encs, atype=[atype for _ in range(num_encs)], han_type=atype)
batch = prepare_inputs("pytorch", num_encs)
model = m.E2E([40 for _ in range(num_encs)], 5, args)
with chainer.no_backprop_mode():
att_ws = model.calculate_all_attentions(*batch)
for i in range(num_encs):
print(att_ws[i][0].shape) # att
print(att_ws[num_encs][0].shape) # han
def test_multi_gpu_trainable(module, num_encs):
m = importlib.import_module(module)
ngpu = 2
device_ids = list(range(ngpu))
args = make_arg(num_encs=num_encs)
model = m.E2E([2 for _ in range(num_encs)], 2, args)
if "pytorch" in module:
model = torch.nn.DataParallel(model, device_ids)
batch = prepare_inputs("pytorch", num_encs, is_cuda=True)
model.cuda()
loss = 1.0 / ngpu * model(*batch)
loss.backward(loss.new_ones(ngpu)) # trainable
def test_sortagrad_trainable(module, num_encs):
args = make_arg(num_encs=num_encs, sortagrad=1)
dummy_json = make_dummy_json(6, [10, 20], [10, 20], idim=20, odim=5, num_inputs=num_encs)
import espnet.nets.pytorch_backend.e2e_asr_mulenc as m
batchset = make_batchset(dummy_json, 2, 2 ** 10, 2 ** 10, shortest_first=True)
model = m.E2E([20 for _ in range(num_encs)], 5, args)
num_utts = 0
for batch in batchset:
num_utts += len(batch)
loss = model(*convert_batch(batch, module, idim=20, odim=5, num_inputs=num_encs))
loss.backward() # trainable
assert num_utts == 6
with torch.no_grad(), chainer.no_backprop_mode():
in_data = [np.random.randn(50, 20) for _ in range(num_encs)]
model.recognize(in_data, args, args.char_list)
def test_calculate_plot_attention_ctc(module, num_encs, model_dict):
args = make_arg(num_encs=num_encs, **model_dict)
m = importlib.import_module(module)
model = m.E2E([2 for _ in range(num_encs)], 2, args)
# test attention plot
dummy_json = make_dummy_json(num_encs, [2, 3], [2, 3],
idim=2,
odim=2,
num_inputs=num_encs)
batchset = make_batchset(dummy_json, 2, 2**10, 2**10, shortest_first=True)
att_ws = model.calculate_all_attentions(*convert_batch(
batchset[0], "pytorch", idim=2, odim=2, num_inputs=num_encs))
from espnet.asr.asr_utils import PlotAttentionReport
tmpdir = tempfile.mkdtemp()
plot = PlotAttentionReport(model.calculate_all_attentions, batchset[0],
tmpdir, None, None, None)
for i in range(num_encs):
# att-encoder
att_w = plot.trim_attention_weight("utt_%d" % 0, att_ws[i][0])
plot._plot_and_save_attention(att_w, "{}/att{}.png".format(tmpdir, i))
# han
att_w = plot.trim_attention_weight("utt_%d" % 0, att_ws[num_encs][0])
plot._plot_and_save_attention(att_w,
"{}/han.png".format(tmpdir),
han_mode=True)
# test CTC plot
ctc_probs = model.calculate_all_ctc_probs(*convert_batch(
batchset[0], "pytorch", idim=2, odim=2, num_inputs=num_encs))
from espnet.asr.asr_utils import PlotCTCReport
tmpdir = tempfile.mkdtemp()
plot = PlotCTCReport(model.calculate_all_ctc_probs, batchset[0], tmpdir,
None, None, None)
if args.mtlalpha > 0:
for i in range(num_encs):
# ctc-encoder
plot._plot_and_save_ctc(ctc_probs[i][0],
"{}/ctc{}.png".format(tmpdir, i))
def test_torch_save_and_load(num_encs):
m = importlib.import_module("espnet.nets.pytorch_backend.e2e_asr_mulenc")
utils = importlib.import_module("espnet.asr.asr_utils")
args = make_arg(num_encs=num_encs)
model = m.E2E([2 for _ in range(num_encs)], 2, args)
# initialize randomly
for p in model.parameters():
p.data.uniform_()
if not os.path.exists(".pytest_cache"):
os.makedirs(".pytest_cache")
tmppath = tempfile.mktemp()
utils.torch_save(tmppath, model)
p_saved = [p.data.numpy() for p in model.parameters()]
# set constant value
for p in model.parameters():
p.data.zero_()
utils.torch_load(tmppath, model)
for p1, p2 in zip(p_saved, model.parameters()):
np.testing.assert_array_equal(p1, p2.data.numpy())
if os.path.exists(tmppath):
os.remove(tmppath)