def test_same_forward_rnn_and_fast_forward_one(self):
wave_rnn = self.wave_rnn
hidden = _make_hidden()
s_one = self.s_one if not wave_rnn.with_speaker else wave_rnn.forward_speaker(self.s_one)
l_array = wave_rnn.forward_encode(l_array=self.l_array, s_one=s_one).data
fast_forward_params = get_fast_forward_params(wave_rnn)
oa, ha = wave_rnn.forward_rnn(
x_array=self.x_array,
l_array=l_array,
hidden=hidden,
)
hb = hidden
for i, (x, l) in enumerate(zip(
np.split(self.x_array, length, axis=1),
np.split(l_array, length, axis=1),
)):
ob, hb = fast_forward_one(
prev_x=x[:, 0],
prev_l=l[:, 0],
hidden=hb,
**fast_forward_params,
)
np.testing.assert_allclose(oa[:, :, i].data, ob.data, atol=1e-6)
np.testing.assert_allclose(ha.data, hb.data, atol=1e-6)
def test_fast_forward_one(self):
wave_rnn = self.wave_rnn
hidden = _make_hidden()
s_one = self.s_one if not wave_rnn.with_speaker else wave_rnn.forward_speaker(self.s_one)
l_one = wave_rnn.forward_encode(l_array=self.l_one, s_one=s_one).data
fast_forward_params = get_fast_forward_params(wave_rnn)
fast_forward_one(
prev_x=self.x_one[:, 0],
prev_l=l_one[:, 0],
hidden=hidden,
**fast_forward_params,
)
def get_fast_forward_params_one(wave_rnn):
fast_forward_params = get_fast_forward_params(wave_rnn)
dtype = fast_forward_params["gru_xb"].dtype
fast_forward_params["w_gru_x"] = xp.empty(
(batch_size, len(fast_forward_params["gru_xb"])), dtype=dtype)
fast_forward_params["w_gru_h"] = xp.empty(
(batch_size, len(fast_forward_params["gru_hb"])), dtype=dtype)
fast_forward_params["w_out_x1"] = xp.empty(
(batch_size, len(fast_forward_params["O1_b"])), dtype=dtype)
fast_forward_params["w_out_x2"] = xp.empty(
(batch_size, len(fast_forward_params["O2_b"])), dtype=dtype)
return fast_forward_params
def __init__(
self,
config: Config,
model: WaveRNN,
max_batch_size: int = 10,
use_cpp_inference: bool = True,
) -> None:
self.config = config
self.model = model
self.max_batch_size = max_batch_size
self.use_cpp_inference = use_cpp_inference
self.sampling_rate = config.dataset.sampling_rate
self.mulaw = config.dataset.mulaw
assert not self.dual_softmax
# setup cpp inference
if use_cpp_inference:
import yukarin_autoreg_cpp
params = get_fast_forward_params(self.model)
local_size = (config.model.conditioning_size * 2
if config.model.conditioning_size is not None else 0)
yukarin_autoreg_cpp.initialize(
graph_length=1000,
max_batch_size=max_batch_size,
local_size=local_size,
hidden_size=config.model.hidden_size,
embedding_size=config.model.embedding_size,
linear_hidden_size=config.model.linear_hidden_size,
output_size=2**config.model.bit_size,
x_embedder_W=to_numpy(params["x_embedder_W"]),
gru_xw=to_numpy(params["gru_xw"]),
gru_xb=to_numpy(params["gru_xb"]),
gru_hw=to_numpy(params["gru_hw"]),
gru_hb=to_numpy(params["gru_hb"]),
O1_W=to_numpy(params["O1_W"]),
O1_b=to_numpy(params["O1_b"]),
O2_W=to_numpy(params["O2_W"]),
O2_b=to_numpy(params["O2_b"]),
)
def test_batchsize1_fast_forward_one(self):
wave_rnn = self.wave_rnn
hidden = _make_hidden()
s_one = self.s_one if not wave_rnn.with_speaker else wave_rnn.forward_speaker(self.s_one)
l_one = wave_rnn.forward_encode(l_array=self.l_one, s_one=s_one).data
fast_forward_params = get_fast_forward_params(wave_rnn)
oa, ha = fast_forward_one(
prev_x=self.x_one[:, 0],
prev_l=l_one[:, 0],
hidden=hidden,
**fast_forward_params,
)
ob, hb = fast_forward_one(
prev_x=self.x_one[:1, 0],
prev_l=l_one[:1, 0],
hidden=hidden[:1],
**fast_forward_params,
)
np.testing.assert_allclose(oa.data[:1], ob.data, atol=1e-6)
np.testing.assert_allclose(ha.data[:1], hb.data, atol=1e-6)
def main_forward(
self,
length: int,
sampling_policy: SamplingPolicy,
num_generate: int,
local_array: np.ndarray = None,
s_one: np.ndarray = None,
):
if self.model.with_local:
with chainer.using_config("train", False), chainer.using_config(
"enable_backprop", False):
local_array = self.model.forward_encode(l_array=local_array,
s_one=s_one).data
c = self.xp.zeros([num_generate], dtype=np.float32)
c = encode_single(c, bit=self.single_bit)
hidden_coarse = self.model.gru.init_hx(local_array)[0].data
if self.use_cpp_inference and sampling_policy == SamplingPolicy.random:
import yukarin_autoreg_cpp
wave = np.zeros((length, num_generate), dtype=np.int32)
yukarin_autoreg_cpp.inference(
batch_size=num_generate,
length=length,
output=wave,
x=to_numpy(c),
l_array=to_numpy(self.xp.transpose(local_array, (1, 0, 2))),
hidden=to_numpy(hidden_coarse),
)
else:
if sampling_policy == SamplingPolicy.random:
fast_forward_params = get_fast_forward_params(self.model)
w_list = fast_generate(
length=length,
x=c,
l_array=local_array,
h=hidden_coarse,
**fast_forward_params,
)
else:
w_list = []
hc = hidden_coarse
for i in tqdm(range(length), desc="generate"):
with chainer.using_config("train",
False), chainer.using_config(
"enable_backprop", False):
c, hc = self.model.forward_one(
prev_x=c,
prev_l=local_array[:, i],
hidden=hc,
)
if sampling_policy == SamplingPolicy.random:
is_random = True
elif sampling_policy == SamplingPolicy.maximum:
is_random = False
else:
raise ValueError(sampling_policy)
c = self.model.sampling(c, maximum=not is_random)
w_list.append(c)
wave = self.xp.stack(w_list)
wave = cuda.to_cpu(wave)
wave = wave.T
wave = decode_single(wave, bit=self.single_bit)
if self.mulaw:
wave = decode_mulaw(wave, mu=2**self.single_bit)
return [
Wave(wave=w_one, sampling_rate=self.sampling_rate)
for w_one in wave
]
def main():
config = ModelConfig(
dual_softmax=False,
bit_size=12,
conditioning_size=None,
embedding_size=16,
hidden_size=896,
linear_hidden_size=896,
local_size=0,
local_scale=None,
local_layer_num=None,
speaker_size=0,
speaker_embedding_size=None,
weight_initializer=None,
)
model = create_predictor(config)
model.to_device(gpu)
xp = model.xp
batch_size = 4
length = 1024
# # non recurrent
# for i in range(10):
# x_array = xp.array(np.random.randint(0, config.bit_size ** 2, size=(batch_size, length + 1)).astype(np.int32))
# l_array = xp.array(np.empty((batch_size, length + 1, 0), dtype=np.float32))
#
# start = time.time()
# with chainer.using_config('train', False), \
# chainer.using_config('enable_backprop', False):
# _, hidden = model(x_array=x_array, l_array=l_array)
# elapsed = time.time() - start
# print(f'non recurrent time: {elapsed}')
# recurrent
for i in range(10):
# # not fast recurrent
# x = xp.array(np.random.randint(0, config.bit_size ** 2, size=(batch_size, )).astype(np.int32))
# l_array = xp.array(np.empty((batch_size, length, 0), dtype=np.float32))
#
# h = None
#
# start = time.time()
# for i in range(length):
# with chainer.using_config('train', False), \
# chainer.using_config('enable_backprop', False):
# x, h = model.forward_one(
# prev_x=x_array[:, i],
# prev_l=l_array[:, i],
# hidden=h,
# )
#
# x = model.sampling(x, maximum=False)
#
# elapsed = time.time() - start
# print(f'not fast recurrent time: {elapsed}')
# fast recurrent
x = xp.array(np.random.randint(0, config.bit_size ** 2, size=(batch_size,)).astype(np.int32))
l_array = xp.array(np.empty((batch_size, length, 0), dtype=np.float32))
h = model.gru.init_hx(l_array)[0].data
fast_forward_params = get_fast_forward_params(model)
# TODO: CPUのときはxp=npを指定しないほうが速い理由を調べる
if gpu != -1:
fast_forward_params['xp'] = xp
start = time.time()
fast_generate(
length=length,
x=x,
l_array=l_array,
h=h,
**fast_forward_params,
)
elapsed = time.time() - start
print(f'fast recurrent time: {elapsed}')
# local_scale=None,
# local_layer_num=None,
# speaker_size=0,
# speaker_embedding_size=None,
# weight_initializer=None,
# )
model = create_predictor(config)
model.to_device(0)
local_size = config.conditioning_size * 2 if config.conditioning_size is not None else 0
base_x = model.xp.array(numpy.random.randint(0, config.bit_size ** 2, size=(max_batch_size,)).astype(numpy.int32))
base_l_array = model.xp.array(numpy.random.rand(length, max_batch_size, local_size).astype(numpy.float32))
base_hidden = model.xp.array(numpy.random.rand(max_batch_size, config.hidden_size).astype(numpy.float32))
params = get_fast_forward_params(model)
def to_numpy(a):
if isinstance(a, cupy.ndarray):
a = cupy.asnumpy(a)
return numpy.ascontiguousarray(a)
# C++
yukarin_autoreg_cpp.initialize(
graph_length=graph_length,
max_batch_size=max_batch_size,
local_size=local_size,
hidden_size=config.hidden_size,
embedding_size=config.embedding_size,
linear_hidden_size=config.linear_hidden_size,
output_size=2 ** config.bit_size,
