def test_sortagrad_trainable_with_batch_bins(module):
args = make_arg(sortagrad=1)
idim = 6
odim = 5
dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=idim, odim=odim)
if module == "pytorch":
import espnet.nets.pytorch_backend.e2e_mt as m
else:
import espnet.nets.chainer_backend.e2e_mt as m
batch_elems = 2000
batchset = make_batchset(dummy_json, batch_bins=batch_elems, shortest_first=True, mt=True, iaxis=1, oaxis=0)
for batch in batchset:
n = 0
for uttid, info in batch:
ilen = int(info['output'][1]['shape'][0])
olen = int(info['output'][0]['shape'][0])
n += ilen * idim + olen * odim
assert olen < batch_elems
model = m.E2E(6, 5, args)
for batch in batchset:
attn_loss = model(*convert_batch(batch, module, idim=6, odim=5))
attn_loss.backward()
with torch.no_grad(), chainer.no_backprop_mode():
in_data = np.random.randint(0, 5, (1, 100))
model.translate(in_data, args, args.char_list)
def test_multi_gpu_trainable(module):
m = importlib.import_module(module)
ngpu = 2
device_ids = list(range(ngpu))
args = make_arg()
model = m.E2E(6, 5, args)
if "pytorch" in module:
model = torch.nn.DataParallel(model, device_ids)
batch = prepare_inputs("pytorch", is_cuda=True)
model.cuda()
loss = 1. / ngpu * model(*batch)
loss.backward(loss.new_ones(ngpu)) # trainable
else:
import copy
import cupy
losses = []
for device in device_ids:
with cupy.cuda.Device(device):
batch = prepare_inputs("chainer", is_cuda=True)
_model = copy.deepcopy(
model
) # Transcribed from training.updaters.ParallelUpdater
_model.to_gpu()
loss = 1. / ngpu * _model(*batch)
losses.append(loss)
for loss in losses:
loss.backward() # trainable
def test_sortagrad_trainable_with_batch_frames(module):
args = make_arg(sortagrad=1)
idim = 6
odim = 5
dummy_json = make_dummy_json_mt(8, [100, 200], [100, 200],
idim=idim,
odim=odim)
if module == "pytorch":
import espnet.nets.pytorch_backend.e2e_mt as m
else:
import espnet.nets.chainer_backend.e2e_mt as m
batch_frames_in = 200
batch_frames_out = 200
batchset = make_batchset(dummy_json,
batch_frames_in=batch_frames_in,
batch_frames_out=batch_frames_out,
shortest_first=True,
mt=True)
for batch in batchset:
i = 0
o = 0
for uttid, info in batch:
i += int(info['output'][1]['shape'][0])
o += int(info['output'][0]['shape'][0])
assert i <= batch_frames_in
assert o <= batch_frames_out
model = m.E2E(6, 5, args)
for batch in batchset:
attn_loss = model(*convert_batch(batch, module, idim=6, odim=5))
attn_loss.backward()
with torch.no_grad(), chainer.no_backprop_mode():
in_data = np.random.randint(0, 5, (1, 100))
model.translate(in_data, args, args.char_list)
def test_sortagrad_trainable(module):
args = make_arg(sortagrad=1)
dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=6, odim=5)
if module == "pytorch":
import espnet.nets.pytorch_backend.e2e_mt as m
else:
import espnet.nets.chainer_backend.e2e_mt as m
batchset = make_batchset(dummy_json,
2,
2**10,
2**10,
shortest_first=True,
mt=True,
iaxis=1,
oaxis=0)
model = m.E2E(6, 5, args)
for batch in batchset:
loss = model(*convert_batch(batch, module, idim=6, odim=5))
if isinstance(loss, tuple):
# chainer return several values as tuple
loss[0].backward() # trainable
else:
loss.backward() # trainable
with torch.no_grad(), chainer.no_backprop_mode():
in_data = np.random.randint(0, 5, (1, 100))
model.translate(in_data, args, args.char_list)
def test_gpu_trainable(module):
m = importlib.import_module(module)
args = make_arg()
model = m.E2E(6, 5, args)
if "pytorch" in module:
batch = prepare_inputs("pytorch", is_cuda=True)
model.cuda()
else:
batch = prepare_inputs("chainer", is_cuda=True)
model.to_gpu()
loss = model(*batch)
loss.backward() # trainable
def test_calculate_all_attentions(module, atype):
m = importlib.import_module(module)
args = make_arg(atype=atype)
if "pytorch" in module:
batch = prepare_inputs("pytorch")
else:
batch = prepare_inputs("chainer")
model = m.E2E(6, 5, args)
with chainer.no_backprop_mode():
if "pytorch" in module:
att_ws = model.calculate_all_attentions(*batch)[0]
else:
att_ws = model.calculate_all_attentions(*batch)
print(att_ws.shape)
def test_multi_gpu_trainable(module):
m = importlib.import_module(module)
ngpu = 2
device_ids = list(range(ngpu))
args = make_arg()
model = m.E2E(6, 5, args)
if "pytorch" in module:
model = torch.nn.DataParallel(model, device_ids)
batch = prepare_inputs("pytorch", is_cuda=True)
model.cuda()
loss = 1.0 / ngpu * model(*batch)
loss.backward(loss.new_ones(ngpu)) # trainable
else:
raise NotImplementedError
def test_sortagrad_trainable(module):
args = make_arg(sortagrad=1)
dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=6, odim=5)
if module == "pytorch":
import espnet.nets.pytorch_backend.e2e_mt as m
else:
import espnet.nets.chainer_backend.e2e_mt as m
batchset = make_batchset(dummy_json, 2, 2 ** 10, 2 ** 10, shortest_first=True, mt=True)
model = m.E2E(6, 5, args)
for batch in batchset:
attn_loss = model(*convert_batch(batch, module, idim=6, odim=5))
attn_loss.backward()
with torch.no_grad(), chainer.no_backprop_mode():
in_data = np.random.randint(0, 5, (1, 100))
model.translate(in_data, args, args.char_list)
def test_gpu_trainable(module):
m = importlib.import_module(module)
args = make_arg()
model = m.E2E(6, 5, args)
if "pytorch" in module:
batch = prepare_inputs("pytorch", is_cuda=True)
model.cuda()
else:
raise NotImplementedError
loss = model(*batch)
if isinstance(loss, tuple):
# chainer return several values as tuple
loss[0].backward() # trainable
else:
loss.backward() # trainable
def test_context_residual(module):
args = make_arg(context_residual=True)
dummy_json = make_dummy_json_mt(8, [1, 100], [1, 100], idim=6, odim=5)
if module == "pytorch":
import espnet.nets.pytorch_backend.e2e_mt as m
else:
raise NotImplementedError
batchset = make_batchset(dummy_json,
2,
2**10,
2**10,
shortest_first=True,
mt=True)
model = m.E2E(6, 5, args)
for batch in batchset:
attn_loss = model(*convert_batch(batch, module, idim=6, odim=5))
attn_loss.backward()
with torch.no_grad(), chainer.no_backprop_mode():
in_data = np.random.randint(0, 5, (1, 100))
model.translate(in_data, args, args.char_list)
def test_torch_save_and_load():
m = importlib.import_module('espnet.nets.pytorch_backend.e2e_mt')
utils = importlib.import_module('espnet.asr.asr_utils')
args = make_arg()
model = m.E2E(6, 5, args)
# initialize randomly
for p in model.parameters():
p.data.uniform_()
if not os.path.exists(".pytest_cache"):
os.makedirs(".pytest_cache")
tmppath = tempfile.mktemp()
utils.torch_save(tmppath, model)
p_saved = [p.data.numpy() for p in model.parameters()]
# set constant value
for p in model.parameters():
p.data.zero_()
utils.torch_load(tmppath, model)
for p1, p2 in zip(p_saved, model.parameters()):
np.testing.assert_array_equal(p1, p2.data.numpy())
if os.path.exists(tmppath):
os.remove(tmppath)
def test_sortagrad_trainable_with_batch_bins(module):
args = make_arg(sortagrad=1)
idim = 6
odim = 5
dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20],
idim=idim,
odim=odim)
if module == "pytorch":
import espnet.nets.pytorch_backend.e2e_mt as m
else:
raise NotImplementedError
batch_elems = 2000
batchset = make_batchset(
dummy_json,
batch_bins=batch_elems,
shortest_first=True,
mt=True,
iaxis=1,
oaxis=0,
)
for batch in batchset:
n = 0
for uttid, info in batch:
ilen = int(info["output"][1]["shape"][0])
olen = int(info["output"][0]["shape"][0])
n += ilen * idim + olen * odim
assert olen < batch_elems
model = m.E2E(6, 5, args)
for batch in batchset:
loss = model(*convert_batch(batch, module, idim=6, odim=5))
if isinstance(loss, tuple):
# chainer return several values as tuple
loss[0].backward() # trainable
else:
loss.backward() # trainable
with torch.no_grad(), chainer.no_backprop_mode():
in_data = np.random.randint(0, 5, (1, 100))
model.translate(in_data, args, args.char_list)
