def __init__(self,
size,
n_cat,
decay=0.9,
eps=2e-5,
dtype=numpy.float32,
initial_gamma=None,
initial_beta=None):
super(CategoricalConditionalBatchNormalization,
self).__init__(size=size,
n_cat=n_cat,
decay=decay,
eps=eps,
dtype=dtype)
with self.init_scope():
if initial_gamma is None:
initial_gamma = 1
initial_gamma = initializers._get_initializer(initial_gamma)
initial_gamma.dtype = dtype
self.gammas = EmbedID(n_cat, size, initialW=initial_gamma)
if initial_beta is None:
initial_beta = 0
initial_beta = initializers._get_initializer(initial_beta)
initial_beta.dtype = dtype
self.betas = EmbedID(n_cat, size, initialW=initial_beta)
def __init__(self, vocab_src_size, vocab_dst_size, bad_word, ds, dt):
super().__init__()
with self.init_scope():
# TODO:
self.emb_src = EmbedID(vocab_src_size, ds, ignore_label=None)
# TODO: .
self.emb_dst = EmbedID(vocab_dst_size, dt, ignore_label=None)
def __init__(
self,
dual_softmax: bool,
bit_size: int,
conditioning_size: int,
embedding_size: int,
hidden_size: int,
linear_hidden_size: int,
local_size: int,
local_scale: int,
local_layer_num: int,
speaker_size: int,
speaker_embedding_size: int,
weight_initializer: Optional[Initializer] = None,
) -> None:
super().__init__()
assert not dual_softmax
self.dual_softmax = dual_softmax
self.bit_size = bit_size
self.local_size = local_size
self.local_scale = local_scale
self.speaker_size = speaker_size
with self.init_scope():
self.speaker_embedder = (EmbedID(
speaker_size,
speaker_embedding_size,
initialW=weight_initializer,
) if self.with_speaker else None)
self.local_gru = (ModifiedNStepBiGRU(
n_layers=local_layer_num,
in_size=local_size +
(speaker_embedding_size if self.with_speaker else 0),
out_size=conditioning_size,
dropout=0,
initialW=weight_initializer,
) if self.with_local else None)
self.x_embedder = EmbedID(
in_size=self.bins,
out_size=embedding_size,
initialW=weight_initializer,
)
in_size = embedding_size + (2 * conditioning_size
if self.with_local else 0)
self.gru = ModifiedNStepGRU(
n_layers=1,
in_size=in_size,
out_size=hidden_size,
dropout=0,
initialW=weight_initializer,
)
self.O1 = L.Linear(hidden_size,
linear_hidden_size,
initialW=weight_initializer)
self.O2 = L.Linear(linear_hidden_size,
self.bins,
initialW=weight_initializer)
def __init__(self, input_size, vocab_size, embed_size, hidden_size):
super(Encoder, self).__init__(xi1=Linear(input_size, embed_size),
ih1=Linear(embed_size, 4 * hidden_size),
hh1=Linear(hidden_size, 4 * hidden_size),
xi2=EmbedID(vocab_size, embed_size),
ih2=Linear(hidden_size + embed_size,
4 * hidden_size),
hh2=Linear(hidden_size, 4 * hidden_size))
def __init__(self, vocab_size, ndim_embedding, num_layers, ndim_h, kernel_size=4, pooling="fo", zoneout=0, dropout=0, wgain=1, densely_connected=True):
super(Model, self).__init__(
embed=EmbedID(vocab_size, ndim_embedding, ignore_label=BLANK),
dense=ConvolutionND(1, ndim_h, vocab_size, ksize=1, stride=1, pad=0)
)
assert num_layers > 0
self.vocab_size = vocab_size
self.ndim_embedding = ndim_embedding
self.num_layers = num_layers
self.ndim_h = ndim_h
self.kernel_size = kernel_size
self.using_dropout = True if dropout > 0 else False
self.dropout = dropout
self.densely_connected = densely_connected
self.add_link("qrnn0", QRNN(ndim_embedding, ndim_h, kernel_size=kernel_size, pooling=pooling, zoneout=zoneout, wgain=wgain))
for i in xrange(1, num_layers):
self.add_link("qrnn{}".format(i), QRNN(ndim_h, ndim_h, kernel_size=kernel_size, pooling=pooling, zoneout=zoneout, wgain=wgain))
class CategoricalConditionalBatchNormalization(ConditionalBatchNormalization):
"""
Conditional Batch Normalization
Args:
size (int or tuple of ints): Size (or shape) of channel
dimensions.
n_cat (int): the number of categories of categorical variable.
decay (float): Decay rate of moving average. It is used on training.
eps (float): Epsilon value for numerical stability.
dtype (numpy.dtype): Type to use in computing.
use_gamma (bool): If ``True``, use scaling parameter. Otherwise, use
unit(1) which makes no effect.
use_beta (bool): If ``True``, use shifting parameter. Otherwise, use
unit(0) which makes no effect.
See: `Batch Normalization: Accelerating Deep Network Training by Reducing\
Internal Covariate Shift <https://arxiv.org/abs/1502.03167>`_
.. seealso::
:func:`~chainer.functions.batch_normalization`,
:func:`~chainer.functions.fixed_batch_normalization`
Attributes:
gamma (~chainer.Variable): Scaling parameter.
beta (~chainer.Variable): Shifting parameter.
avg_mean (numpy.ndarray or cupy.ndarray): Population mean.
avg_var (numpy.ndarray or cupy.ndarray): Population variance.
N (int): Count of batches given for fine-tuning.
decay (float): Decay rate of moving average. It is used on training.
eps (float): Epsilon value for numerical stability. This value is added
to the batch variances.
"""
def __init__(self,
size,
n_cat,
decay=0.9,
eps=2e-5,
dtype=numpy.float32,
initial_gamma=None,
initial_beta=None):
super(CategoricalConditionalBatchNormalization,
self).__init__(size=size,
n_cat=n_cat,
decay=decay,
eps=eps,
dtype=dtype)
with self.init_scope():
if initial_gamma is None:
initial_gamma = 1
initial_gamma = initializers._get_initializer(initial_gamma)
initial_gamma.dtype = dtype
self.gammas = EmbedID(n_cat, size, initialW=initial_gamma)
if initial_beta is None:
initial_beta = 0
initial_beta = initializers._get_initializer(initial_beta)
initial_beta.dtype = dtype
self.betas = EmbedID(n_cat, size, initialW=initial_beta)
def __call__(self, x, c, finetune=False, **kwargs):
"""__call__(self, x, c, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluatino during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', train)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
c (Variable): Input variable for conditioning gamma and beta
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
weights, = argument.parse_kwargs(kwargs, ('weights', None))
if c.ndim == 2 and weights is not None:
_gamma_c = self.gammas(c)
_beta_c = self.betas(c)
_gamma_c = F.broadcast_to(F.expand_dims(weights, 2),
_gamma_c.shape) * _gamma_c
_beta_c = F.broadcast_to(F.expand_dims(weights, 2),
_beta_c.shape) * _beta_c
gamma_c = F.sum(_gamma_c, 1)
beta_c = F.sum(_beta_c, 1)
else:
if c.ndim == 1: # just labels
gamma_c = self.gammas(c)
beta_c = self.betas(c)
else: # distributions
wg = list(self.gammas.params())[0]
wb = list(self.betas.params())[0]
gamma_c = chainer.functions.matmul(c, wg)
beta_c = chainer.functions.matmul(c, wb)
return super(CategoricalConditionalBatchNormalization,
self).__call__(x, gamma_c, beta_c, **kwargs)
