def test_sortagrad_trainable_with_batch_bins(module): args = make_arg(sortagrad=1) idim = 20 odim = 5 dummy_json = make_dummy_json_st( 4, [10, 20], [10, 20], [10, 20], idim=idim, odim=odim ) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_st as m else: raise NotImplementedError batch_elems = 2000 batchset = make_batchset(dummy_json, batch_bins=batch_elems, shortest_first=True) for batch in batchset: n = 0 for uttid, info in batch: ilen = int(info["input"][0]["shape"][0]) olen = int(info["output"][0]["shape"][0]) n += ilen * idim + olen * odim assert olen < batch_elems model = m.E2E(20, 5, args) for batch in batchset: loss = model(*convert_batch(batch, module, idim=20, odim=5)) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randn(100, 20) model.translate(in_data, args, args.char_list)
def test_gradient_noise_injection(module): args = make_arg(grad_noise=True) args_org = make_arg() dummy_json = make_dummy_json_st(2, [10, 20], [10, 20], [10, 20], idim=20, odim=5) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_st as m else: raise NotImplementedError batchset = make_batchset(dummy_json, 2, 2**10, 2**10, shortest_first=True) model = m.E2E(20, 5, args) model_org = m.E2E(20, 5, args_org) for batch in batchset: loss = model(*convert_batch(batch, module, idim=20, odim=5)) loss_org = model_org(*convert_batch(batch, module, idim=20, odim=5)) loss.backward() grad = [param.grad for param in model.parameters()][10] loss_org.backward() grad_org = [param.grad for param in model_org.parameters()][10] assert grad[0] != grad_org[0]
def test_multi_gpu_trainable(module): m = importlib.import_module(module) ngpu = 2 device_ids = list(range(ngpu)) args = make_arg() model = m.E2E(40, 5, args) if "pytorch" in module: model = torch.nn.DataParallel(model, device_ids) batch = prepare_inputs("pytorch", is_cuda=True) model.cuda() loss = 1.0 / ngpu * model(*batch) loss.backward(loss.new_ones(ngpu)) # trainable else: raise NotImplementedError
def test_calculate_all_attentions(module, atype): m = importlib.import_module(module) args = make_arg(atype=atype) if "pytorch" in module: batch = prepare_inputs("pytorch") else: raise NotImplementedError model = m.E2E(40, 5, args) with chainer.no_backprop_mode(): if "pytorch" in module: att_ws = model.calculate_all_attentions(*batch)[0] else: raise NotImplementedError print(att_ws.shape)
def test_gpu_trainable(module): m = importlib.import_module(module) args = make_arg() model = m.E2E(40, 5, args) if "pytorch" in module: batch = prepare_inputs("pytorch", is_cuda=True) model.cuda() else: raise NotImplementedError loss = model(*batch) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable
def test_calculate_all_ctc_probs(module, mtlalpha): m = importlib.import_module(module) args = make_arg(mtlalpha=mtlalpha, asr_weight=0.3) if "pytorch" in module: batch = prepare_inputs("pytorch") else: batch = prepare_inputs("chainer") model = m.E2E(40, 5, args) with chainer.no_backprop_mode(): if "pytorch" in module: ctc_probs = model.calculate_all_ctc_probs(*batch) if mtlalpha > 0: print(ctc_probs.shape) else: assert ctc_probs is None else: raise NotImplementedError
def test_sortagrad_trainable(module): args = make_arg(sortagrad=1) dummy_json = make_dummy_json_st(4, [10, 20], [10, 20], [10, 20], idim=20, odim=5) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_st as m else: raise NotImplementedError batchset = make_batchset(dummy_json, 2, 2**10, 2**10, shortest_first=True) model = m.E2E(20, 5, args) for batch in batchset: loss = model(*convert_batch(batch, module, idim=20, odim=5)) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randn(50, 20) model.translate(in_data, args, args.char_list)
def test_torch_save_and_load(): m = importlib.import_module("espnet.nets.pytorch_backend.e2e_st") utils = importlib.import_module("espnet.asr.asr_utils") args = make_arg() model = m.E2E(40, 5, args) # initialize randomly for p in model.parameters(): p.data.uniform_() if not os.path.exists(".pytest_cache"): os.makedirs(".pytest_cache") tmppath = tempfile.mktemp() utils.torch_save(tmppath, model) p_saved = [p.data.numpy() for p in model.parameters()] # set constant value for p in model.parameters(): p.data.zero_() utils.torch_load(tmppath, model) for p1, p2 in zip(p_saved, model.parameters()): np.testing.assert_array_equal(p1, p2.data.numpy()) if os.path.exists(tmppath): os.remove(tmppath)
def test_sortagrad_trainable_with_batch_frames(module): args = make_arg(sortagrad=1) idim = 20 odim = 5 dummy_json = make_dummy_json_st(4, [10, 20], [10, 20], [10, 20], idim=idim, odim=odim) if module == "pytorch": import espnet.nets.pytorch_backend.e2e_st as m else: raise NotImplementedError batch_frames_in = 50 batch_frames_out = 50 batchset = make_batchset(dummy_json, batch_frames_in=batch_frames_in, batch_frames_out=batch_frames_out, shortest_first=True) for batch in batchset: i = 0 o = 0 for uttid, info in batch: i += int(info['input'][0]['shape'][0]) o += int(info['output'][0]['shape'][0]) assert i <= batch_frames_in assert o <= batch_frames_out model = m.E2E(20, 5, args) for batch in batchset: loss = model(*convert_batch(batch, module, idim=20, odim=5)) if isinstance(loss, tuple): # chainer return several values as tuple loss[0].backward() # trainable else: loss.backward() # trainable with torch.no_grad(), chainer.no_backprop_mode(): in_data = np.random.randn(100, 20) model.translate(in_data, args, args.char_list)