def test_sortagrad_trainable_with_batch_bins(module): args = make_arg(sortagrad=1) idim = 6 odim = 5 dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=idim, odim=odim) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_mt as m else: import espnet.nets.chainer_backend.e2e_mt as m batch_elems = 2000 batchset = make_batchset(dummy_json, batch_bins=batch_elems, shortest_first=True, mt=True, iaxis=1, oaxis=0) for batch in batchset: n = 0 for uttid, info in batch: ilen = int(info['output'][1]['shape'][0]) olen = int(info['output'][0]['shape'][0]) n += ilen * idim + olen * odim assert olen < batch_elems model = m.E2E(6, 5, args) for batch in batchset: attn_loss = model(*convert_batch(batch, module, idim=6, odim=5)) attn_loss.backward() with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randint(0, 5, (1, 100)) model.translate(in_data, args, args.char_list)
def test_multi_gpu_trainable(module): m = importlib.import_module(module) ngpu = 2 device_ids = list(range(ngpu)) args = make_arg() model = m.E2E(6, 5, args) if "pytorch" in module: model = torch.nn.DataParallel(model, device_ids) batch = prepare_inputs("pytorch", is_cuda=True) model.cuda() loss = 1. / ngpu * model(*batch) loss.backward(loss.new_ones(ngpu)) # trainable else: import copy import cupy losses = [] for device in device_ids: with cupy.cuda.Device(device): batch = prepare_inputs("chainer", is_cuda=True) _model = copy.deepcopy( model ) # Transcribed from training.updaters.ParallelUpdater _model.to_gpu() loss = 1. / ngpu * _model(*batch) losses.append(loss) for loss in losses: loss.backward() # trainable
def test_sortagrad_trainable_with_batch_frames(module): args = make_arg(sortagrad=1) idim = 6 odim = 5 dummy_json = make_dummy_json_mt(8, [100, 200], [100, 200], idim=idim, odim=odim) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_mt as m else: import espnet.nets.chainer_backend.e2e_mt as m batch_frames_in = 200 batch_frames_out = 200 batchset = make_batchset(dummy_json, batch_frames_in=batch_frames_in, batch_frames_out=batch_frames_out, shortest_first=True, mt=True) for batch in batchset: i = 0 o = 0 for uttid, info in batch: i += int(info['output'][1]['shape'][0]) o += int(info['output'][0]['shape'][0]) assert i <= batch_frames_in assert o <= batch_frames_out model = m.E2E(6, 5, args) for batch in batchset: attn_loss = model(*convert_batch(batch, module, idim=6, odim=5)) attn_loss.backward() with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randint(0, 5, (1, 100)) model.translate(in_data, args, args.char_list)
def test_sortagrad_trainable(module): args = make_arg(sortagrad=1) dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=6, odim=5) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_mt as m else: import espnet.nets.chainer_backend.e2e_mt as m batchset = make_batchset(dummy_json, 2, 2**10, 2**10, shortest_first=True, mt=True, iaxis=1, oaxis=0) model = m.E2E(6, 5, args) for batch in batchset: loss = model(*convert_batch(batch, module, idim=6, odim=5)) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randint(0, 5, (1, 100)) model.translate(in_data, args, args.char_list)
def test_gpu_trainable(module): m = importlib.import_module(module) args = make_arg() model = m.E2E(6, 5, args) if "pytorch" in module: batch = prepare_inputs("pytorch", is_cuda=True) model.cuda() else: batch = prepare_inputs("chainer", is_cuda=True) model.to_gpu() loss = model(*batch) loss.backward() # trainable
def test_calculate_all_attentions(module, atype): m = importlib.import_module(module) args = make_arg(atype=atype) if "pytorch" in module: batch = prepare_inputs("pytorch") else: batch = prepare_inputs("chainer") model = m.E2E(6, 5, args) with chainer.no_backprop_mode(): if "pytorch" in module: att_ws = model.calculate_all_attentions(*batch)[0] else: att_ws = model.calculate_all_attentions(*batch) print(att_ws.shape)
def test_multi_gpu_trainable(module): m = importlib.import_module(module) ngpu = 2 device_ids = list(range(ngpu)) args = make_arg() model = m.E2E(6, 5, args) if "pytorch" in module: model = torch.nn.DataParallel(model, device_ids) batch = prepare_inputs("pytorch", is_cuda=True) model.cuda() loss = 1.0 / ngpu * model(*batch) loss.backward(loss.new_ones(ngpu)) # trainable else: raise NotImplementedError
def test_sortagrad_trainable(module): args = make_arg(sortagrad=1) dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=6, odim=5) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_mt as m else: import espnet.nets.chainer_backend.e2e_mt as m batchset = make_batchset(dummy_json, 2, 2 ** 10, 2 ** 10, shortest_first=True, mt=True) model = m.E2E(6, 5, args) for batch in batchset: attn_loss = model(*convert_batch(batch, module, idim=6, odim=5)) attn_loss.backward() with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randint(0, 5, (1, 100)) model.translate(in_data, args, args.char_list)
def test_gpu_trainable(module): m = importlib.import_module(module) args = make_arg() model = m.E2E(6, 5, args) if "pytorch" in module: batch = prepare_inputs("pytorch", is_cuda=True) model.cuda() else: raise NotImplementedError loss = model(*batch) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable
def test_context_residual(module): args = make_arg(context_residual=True) dummy_json = make_dummy_json_mt(8, [1, 100], [1, 100], idim=6, odim=5) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_mt as m else: raise NotImplementedError batchset = make_batchset(dummy_json, 2, 2**10, 2**10, shortest_first=True, mt=True) model = m.E2E(6, 5, args) for batch in batchset: attn_loss = model(*convert_batch(batch, module, idim=6, odim=5)) attn_loss.backward() with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randint(0, 5, (1, 100)) model.translate(in_data, args, args.char_list)
def test_torch_save_and_load(): m = importlib.import_module('espnet.nets.pytorch_backend.e2e_mt') utils = importlib.import_module('espnet.asr.asr_utils') args = make_arg() model = m.E2E(6, 5, args) # initialize randomly for p in model.parameters(): p.data.uniform_() if not os.path.exists(".pytest_cache"): os.makedirs(".pytest_cache") tmppath = tempfile.mktemp() utils.torch_save(tmppath, model) p_saved = [p.data.numpy() for p in model.parameters()] # set constant value for p in model.parameters(): p.data.zero_() utils.torch_load(tmppath, model) for p1, p2 in zip(p_saved, model.parameters()): np.testing.assert_array_equal(p1, p2.data.numpy()) if os.path.exists(tmppath): os.remove(tmppath)
def test_sortagrad_trainable_with_batch_bins(module): args = make_arg(sortagrad=1) idim = 6 odim = 5 dummy_json = make_dummy_json_mt(4, [10, 20], [10, 20], idim=idim, odim=odim) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_mt as m else: raise NotImplementedError batch_elems = 2000 batchset = make_batchset( dummy_json, batch_bins=batch_elems, shortest_first=True, mt=True, iaxis=1, oaxis=0, ) for batch in batchset: n = 0 for uttid, info in batch: ilen = int(info["output"][1]["shape"][0]) olen = int(info["output"][0]["shape"][0]) n += ilen * idim + olen * odim assert olen < batch_elems model = m.E2E(6, 5, args) for batch in batchset: loss = model(*convert_batch(batch, module, idim=6, odim=5)) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randint(0, 5, (1, 100)) model.translate(in_data, args, args.char_list)