def __init__(self,
in_dim,
hidden_dim,
out_dim,
num_mixture,
auto_regressive=False):
self.sampling_bias = 1
self.IN_DIM = in_dim
self.HIDDEN_DIM = hidden_dim
self.OUT_DIM = out_dim
self.NUM_MIXTURE = num_mixture
self.AUTO_REGRESSIVE = auto_regressive
if auto_regressive:
self.future_out_dim = 1
else:
self.future_out_dim = out_dim
super(RobotController, self).__init__(
ln1_=L.LayerNormalization(),
ln2_=L.LayerNormalization(),
ln3_=L.LayerNormalization(),
l1_=L.LSTM(in_dim + 4, hidden_dim),
l2_=L.LSTM(hidden_dim + in_dim + 4, hidden_dim),
l3_=L.LSTM(hidden_dim + in_dim + 4, hidden_dim),
# FC1_=L.Linear(hidden_dim, self.future_out_dim)
mixing_=L.Linear(3 * hidden_dim, num_mixture),
mu_=L.Linear(3 * hidden_dim, num_mixture * self.future_out_dim),
sigma_=L.Linear(3 * hidden_dim, num_mixture))
def __init__(self, n_units, h=8, dropout=0.1, nopad=False):
n_inner_units = n_units * 2
super(EncoderLayer, self).__init__(
W_1=L.Linear(n_units, n_inner_units),
W_2=L.Linear(n_inner_units, n_units),
SelfAttention=AttentionLayer(n_units, h),
LN_1=L.LayerNormalization(n_units),
LN_2=L.LayerNormalization(n_units),
)
self.dropout = dropout
def _create_ln(*args, **kwargs):
flag = chainer.disable_experimental_feature_warning
chainer.disable_experimental_feature_warning = True
try:
return links.LayerNormalization(*args, **kwargs)
finally:
chainer.disable_experimental_feature_warning = flag
def __init__(self, in_size, out_size, dropout):
super(LSTM_normalization, self).__init__()
initializer = chainer.initializers.Normal()
with self.init_scope():
self.lstm = L.NStepLSTM(1, in_size, out_size, dropout)
self.layer_norm = L.LayerNormalization(out_size,
eps=1e-5,
initial_gamma=initializer)
def __init__(self, ch=128):
super(Discriminator, self).__init__()
self.ch = ch
self.up_sample_dim = ((4, 4, 2))
self.out_size = self.xp.prod(self.up_sample_dim)
with self.init_scope():
w = chainer.initializers.HeNormal(0.02)
self.c1 = L.Convolution3D(3,
ch // 8,
ksize=4,
stride=2,
pad=1,
initialW=w)
self.c2 = L.Convolution3D(ch // 8,
ch // 4,
ksize=4,
stride=2,
pad=1,
initialW=w)
self.c3 = L.Convolution3D(ch // 4,
ch // 2,
ksize=4,
stride=2,
pad=1,
initialW=w)
self.c4 = L.Convolution3D(ch // 2,
ch,
ksize=4,
stride=2,
pad=1,
initialW=w)
self.c5 = L.Convolution3D(ch,
1,
ksize=3,
stride=1,
pad=1,
initialW=w)
self.l1 = L.Linear(self.out_size, 1, initialW=w)
self.ln2 = L.LayerNormalization(
) # all have layer norm except first and last layer
self.ln3 = L.LayerNormalization()
self.ln4 = L.LayerNormalization()
def __init__(self, n_hidden=128, bottom_width=4, ch=512, wscale=0.02):
super(Generator, self).__init__()
self.n_hidden = n_hidden
self.ch = ch
self.bottom_width = bottom_width
with self.init_scope():
w = chainer.initializers.Normal(wscale)
self.l0 = L.Linear(self.n_hidden, bottom_width * bottom_width * ch,
initialW=w)
self.dc1 = L.Deconvolution2D(ch, ch // 2, 4, 2, 1, initialW=w)
self.dc2 = L.Deconvolution2D(ch // 2, ch // 4, 4, 2, 1, initialW=w)
self.dc3 = L.Deconvolution2D(ch // 4, ch // 8, 4, 2, 1, initialW=w)
self.dc4 = L.Deconvolution2D(ch // 8, 3, 3, 1, 1, initialW=w)
self.bn0 = L.LayerNormalization(bottom_width * bottom_width * ch)
self.bn1 = L.LayerNormalization(ch // 2)
self.bn2 = L.LayerNormalization(ch // 4)
self.bn3 = L.LayerNormalization(ch // 8)
def __init__(self, n_channel, ratio=4):
super(GCBlock, self).__init__()
reduction_size = n_channel // ratio
with self.init_scope():
self.context = L.Convolution2D(n_channel, 1, ksize=1, nobias=True)
self.down = L.Linear(n_channel, reduction_size)
self.ln = L.LayerNormalization(reduction_size)
self.up = L.Linear(reduction_size, n_channel)
def __init__(self, ch0, ch1, \
nn='conv', \
norm='bn', \
activation=F.relu, \
dropout=False, \
noise=None, \
w_init=None, \
k_size = 3, \
normalize_input=False ):
self.norm = norm
self.normalize_input = normalize_input
self.activation = activation
self.dropout = dropout
self.noise = noise
self.nn = nn
layers = {}
if w_init == None:
w = chainer.initializers.GlorotNormal()
else:
w = w_init
if nn == 'down_conv':
layers['c'] = L.Convolution2D(ch0, ch1, 4, 2, 1, initialW=w)
elif nn == 'up_deconv':
layers['c'] = L.Deconvolution2D(ch0, ch1, 4, 2, 1, initialW=w)
elif nn == 'up_subpixel':
pad = k_size//2
layers['c'] = L.Convolution2D(ch0, ch1*4, k_size, 1, pad, initialW=w)
elif nn=='conv' or nn=='up_unpooling':
pad = k_size//2
layers['c'] = L.Convolution2D(ch0, ch1, k_size, 1, pad, initialW=w)
elif nn=='linear':
layers['c'] = L.Linear(ch0, ch1, initialW=w)
else:
raise Exception("Cannot find method %s" % nn)
if self.norm == 'bn':
if self.noise:
layers['n'] = L.BatchNormalization(ch1, use_gamma=False)
else:
layers['n'] = L.BatchNormalization(ch1)
elif self.norm == 'ln':
layers['n'] = L.LayerNormalization(ch1)
super(NNBlock, self).__init__(**layers)
def __init__(self,
idim,
n_layers,
n_units,
e_units=2048,
h=8,
dropout=0.1):
super(TransformerEncoder, self).__init__()
with self.init_scope():
self.linear_in = L.Linear(idim, n_units)
self.lnorm_in = L.LayerNormalization(n_units)
self.pos_enc = PositionalEncoding(n_units, dropout, 5000)
self.n_layers = n_layers
self.dropout = dropout
for i in range(n_layers):
setattr(self, '{}{:d}'.format("lnorm1_", i),
L.LayerNormalization(n_units))
setattr(self, '{}{:d}'.format("self_att_", i),
MultiHeadSelfAttention(n_units, h))
setattr(self, '{}{:d}'.format("lnorm2_", i),
L.LayerNormalization(n_units))
setattr(self, '{}{:d}'.format("ff_", i),
PositionwiseFeedForward(n_units, e_units, dropout))
self.lnorm_out = L.LayerNormalization(n_units)
def __init__(self,
hidden_channels=16,
n_edge_types=5,
activation=functions.relu):
super(GNNFiLMUpdate, self).__init__()
self.n_edge_types = n_edge_types
self.activation = activation
with self.init_scope():
self.W_linear = GraphLinear(in_size=None,
out_size=self.n_edge_types *
hidden_channels,
nobias=True) # W_l in eq. (6)
self.W_g = GraphLinear(in_size=None,
out_size=self.n_edge_types *
hidden_channels * 2,
nobias=True) # g in eq. (6)
self.norm_layer = links.LayerNormalization() # l in eq. (6)
def __init__(self,
n_lstm_layers,
n_mid_units,
n_out,
win_size,
batch_size,
att_units_size,
frame_level=True,
dropout=0.5):
super(RNN, self).__init__()
### actual number of lstm layers is 2*n_lstm_layers ###
initializer = chainer.initializers.Normal()
self.batch_size = batch_size
###### local attention related #####
xp = cuda.cupy
self.Zu_init = xp.zeros((batch_size, n_out), dtype=np.float32)
self.pad_size = int((win_size - 1) / 2)
self.pad_zero = xp.zeros((self.pad_size, n_mid_units),
dtype=np.float32)
self.pad_inf = xp.full((self.pad_size, 1), -1e20, dtype=np.float32)
self.win_size = win_size
self.att_size = 1 if frame_level else n_mid_units
########################################
with self.init_scope():
self.l1 = L.Linear(None, n_mid_units, initialW=initializer)
self.encoder1 = L.NStepLSTM(1, n_mid_units, n_mid_units, dropout)
self.encoder2 = L.NStepLSTM(1, n_mid_units, n_mid_units, dropout)
self.encoder3 = L.NStepLSTM(1, n_mid_units, n_mid_units, dropout)
self.lstm2 = L.NStepLSTM(n_lstm_layers - 3, n_mid_units,
n_mid_units, dropout)
self.attend = Additive_Attention(n_mid_units, win_size, batch_size,
att_units_size)
self.attend_ln = L.LayerNormalization(n_mid_units,
initial_gamma=initializer)
self.output = L.Linear(n_mid_units * 2,
n_out,
initialW=initializer)
def __init__(self,
in_size,
n_units1,
n_units2,
n_out=1,
out_size=3,
train=True,
batch_norm=False,
layer_norm=False):
super(LSTM, self).__init__(
l1=L.Linear(in_size, n_units1),
bn1=L.BatchNormalization(n_units1),
l2=L.LSTM(n_units1, n_units2),
ln1=L.LayerNormalization(n_units2),
out=MultiOut(n_units2, n_out, out_size),
)
self.train = train
self.target_num = n_out
self.batch_norm = batch_norm
self.layer_norm = layer_norm
def build_mlp(n_out,
n_units=256,
layers=5,
normalize=None,
activation='leaky_relu',
dropout=0.0):
net = chainer.Sequential()
if normalize == 'BN' or normalize == 'LN':
nobias = True
else:
nobias = False
for _ in range(layers):
net.append(L.Linear(n_units, nobias=nobias))
if normalize == 'BN':
net.append(L.BatchNormalization(n_units))
elif normalize == 'LN':
net.append(L.LayerNormalization())
net.append(mF.get_function(activation))
net.append(partial(F.dropout, ratio=dropout))
net.append(L.Linear(n_out))
return net
def __init__(self, bottom_width=4, ch=512, wscale=0.02):
w = chainer.initializers.Normal(wscale)
super(Discriminator, self).__init__()
with self.init_scope():
self.c0_0 = L.Convolution2D(3, ch // 8, 3, 1, 1, initialW=w)
self.c0_1 = L.Convolution2D(ch // 8, ch // 4, 4, 2, 1, initialW=w)
self.c1_0 = L.Convolution2D(ch // 4, ch // 4, 3, 1, 1, initialW=w)
self.c1_1 = L.Convolution2D(ch // 4, ch // 2, 4, 2, 1, initialW=w)
self.c2_0 = L.Convolution2D(ch // 2, ch // 2, 3, 1, 1, initialW=w)
self.c2_1 = L.Convolution2D(ch // 2, ch // 1, 4, 2, 1, initialW=w)
self.c3_0 = L.Convolution2D(ch // 1, ch // 1, 3, 1, 1, initialW=w)
self.l4 = L.Linear(bottom_width * bottom_width * ch, 1, initialW=w)
self.bn0_1 = L.LayerNormalization(ch // 4)
self.bn1_0 = L.LayerNormalization(ch // 4)
self.bn1_1 = L.LayerNormalization(ch // 2)
self.bn2_0 = L.LayerNormalization(ch // 2)
self.bn2_1 = L.LayerNormalization(ch // 1)
self.bn3_0 = L.LayerNormalization(ch // 1)
def __init__(self, sublayer, N):
super().__init__()
with self.init_scope():
self.sub_layers = sublayer.repeat(N, mode='copy')
self.norm = L.LayerNormalization(sublayer.size)
def __init__(self, layer, size, dropout_ratio=0.1):
super().__init__()
self.dropout_ratio = dropout_ratio
with self.init_scope():
self.layer = layer
self.norm = L.LayerNormalization(size)
def init_enc_dec_attn(self, RNN_CONFIG):
xp = cuda.cupy if self.gpuid >= 0 else np
"""
Add encoder RNN layers
if bi-rnn, then hidden units in each direction = hidden units / 2
"""
self.rnn_enc = [
"L{0:d}_enc".format(i) for i in range(RNN_CONFIG['enc_layers'])
]
self.bi_rnn = RNN_CONFIG['bi_rnn']
if RNN_CONFIG['bi_rnn']:
enc_lstm_units = RNN_CONFIG['hidden_units'] // 2
self.rnn_rev_enc = [
"L{0:d}_rev_enc".format(i)
for i in range(RNN_CONFIG['enc_layers'])
]
else:
enc_lstm_units = RNN_CONFIG['hidden_units']
self.rnn_rev_enc = []
# Add each layer
self.rnn_ln = RNN_CONFIG['ln']
for i, rnn_name in enumerate(self.rnn_enc + self.rnn_rev_enc):
self.add_link(rnn_name, L.LSTM(None, enc_lstm_units))
# Add layer normalization
if RNN_CONFIG['ln']:
self.add_link("{0:s}_ln".format(rnn_name),
L.LayerNormalization(enc_lstm_units))
# end for enc rnn
# Add linear projection with Batch Norm
self.rnn_linear_proj = False
if 'linear_proj' in RNN_CONFIG and RNN_CONFIG['linear_proj']:
self.rnn_linear_proj = True
proj_units = RNN_CONFIG['hidden_units']
# self.add_link(f"enc_proj", L.Linear(proj_units, proj_units))
# self.add_link(f"enc_proj_bn", L.BatchNormalization((proj_units)))
for i in range(RNN_CONFIG['enc_layers'] - 1):
self.add_link(f"enc_proj{i}", L.Linear(proj_units, proj_units))
self.add_link(f"enc_proj{i}_bn",
L.BatchNormalization((proj_units)))
print(f"RNN linear projection layer: {self.rnn_linear_proj}")
"""
Add attention layers
"""
a_units = RNN_CONFIG['attn_units']
self.add_link(
"attn_Wa",
L.Linear(RNN_CONFIG['hidden_units'], RNN_CONFIG['hidden_units']))
self.n_attn = 1 if 'n_attn' not in RNN_CONFIG else RNN_CONFIG['n_attn']
if self.n_attn > 1:
print(f"# Attention layers: {self.n_attn}")
feed_attn = True if 'feed_attn' not in self.cfg["rnn_config"] else \
self.cfg["rnn_config"]['feed_attn']
print(f"Feed attention to next decode step: {feed_attn}")
for i in range(1, self.n_attn):
self.add_link(
f"attn_Wa{i}",
L.Linear(RNN_CONFIG['hidden_units'],
RNN_CONFIG['hidden_units']))
# Context layer = 1*h_units from enc + 1*h_units from dec
# self.add_link("context", L.Linear(2*RNN_CONFIG['hidden_units'],
# a_units))
self.add_link(
"context",
L.Linear((self.n_attn + 1) * RNN_CONFIG['hidden_units'], a_units))
"""
Add decoder layers
Embedding layer
"""
e_units = RNN_CONFIG['embedding_units']
self.add_link("embed_dec",
L.EmbedID(RNN_CONFIG["dec_vocab_size"], e_units))
# Add decoder rnns
self.rnn_dec = [
"L{0:d}_dec".format(i) for i in range(RNN_CONFIG['dec_layers'])
]
# decoder rnn input = emb + prev. context vector
dec_lstm_units = RNN_CONFIG['hidden_units']
for i, rnn_name in enumerate(self.rnn_dec):
self.add_link(rnn_name, L.LSTM(None, dec_lstm_units))
# Add layer normalization
if RNN_CONFIG['ln']:
self.add_link("{0:s}_ln".format(rnn_name),
L.LayerNormalization(RNN_CONFIG['hidden_units']))
# end for
"""
Add output layers
"""
self.add_link("out", L.Linear(a_units, RNN_CONFIG["dec_vocab_size"]))
# create masking array for pad id
with cupy.cuda.Device(self.gpuid):
self.mask_pad_id = xp.ones(RNN_CONFIG["dec_vocab_size"],
dtype=xp.float32)
# set PAD ID to 0, so as to not compute any loss for it
self.mask_pad_id[0] = 0
def __init__(self, config):
super(LayerNormalization3D, self).__init__()
with self.init_scope():
self.ln = L.LayerNormalization(config.unit_num)
def __init__(self):
super(LayerNormalizationConv2D, self).__init__()
with self.init_scope():
self.norm = L.LayerNormalization()
