def forward(self, x):
for ii in range(0, len(self._convs)):
#x = F.leaky_relu( self._convs[ii](x) )
#x = self._batchs2d[ii](x)
#x = F.dropout2d( x, p=self._p_conv )
x = F.selu(self._convs[ii](x))
x = self._batchs2d[ii](
x) # seems to work better with batch norm ¯\_(ツ)_/¯
x = F.alpha_dropout(x, p=self._p_conv)
if self._transfer:
with torch.no_grad():
u = self._us
v = self._vs
for ii in range(0, len(self._n_convs)):
w = self._convs[ii].weight
self._n_convs[ii].set_weights(w)
u = self._n_convs[ii](u)
v = self._n_convs[ii](v)
# 1st: just concatenate
#x = torch.cat( ( x, u, v ), 1 )
# 2nd: attempt at using 1x1 convs on each channel of [x,u,v] images
#n, c, h, w = x.shape
#x_merged = torch.zeros( ( n, 2*c, h, w ),
# dtype=x.dtype,
# device=torch.device('cuda:0'),
# requires_grad=True )
#print( 'x_merged shape ', x_merged.shape )
#for ii in range( 0, c ):
# temp = torch.cat( ( x[:,ii:ii+1,:,:], u[:,ii:ii+1,:,:], v[:,ii:ii+1,:,:] ), 1 )
# print( 'temp shape ', temp.shape )
# print( 'ii ', ii )
# #x_merged[ :, 2*ii:2*ii+1, :, : ] = self._merge_convs[2*ii]( temp )
# #x_merged[ :, 2*ii+1:2*ii+2, :, : ] = self._merge_convs[2*ii+1]( temp )
# x_merged[ :, 2*ii:2*ii+2, :, : ] = self._merge_convs[ii]( temp )
# 3rd: concatenate and merge
x = torch.cat((x, u, v), 1)
x = self._merge_conv(x)
x = x.view(-1, self._n)
n_in = self._capacity - 1
for ii in range(0, n_in):
#x = F.leaky_relu( self._fcs_in[ii](x) )
#if ( ii != n_in-1 ) :
#x = self._batchs1d[ii](x)
#x = F.dropout( x, p=self._p_fc )
x = F.selu(self._fcs_in[ii](x))
if (ii != n_in - 1):
x = self._batchs1d[ii](x)
x = F.alpha_dropout(x, p=self._p_fc)
x = self._fcs_out[n_in](x)
return x
def forward(self, x, y, z, w):
x = F.alpha_dropout(x, training=False)
y = F.alpha_dropout(y, training=False)
z = F.alpha_dropout(z, p=0.6, training=False)
w = F.alpha_dropout(w, p=0.1, training=False)
x = F.relu(x)
y = F.relu(y)
z = F.relu(z)
w = F.relu(w)
return x, y, z, w
def forward(self, x):
"""
Feed forward algorithm
:param x:
Input
:return: Configured neural network
"""
if self.params.get('initializer') is None:
self.params.update(
dict(initializer=np.random.choice(a=list(INITIALIZER.keys()))))
set_initializer(x=x, **self.params)
x = x.float()
for l, layer in enumerate(self.hidden_layers):
x = self.activation_functions[l](layer(x))
if self.use_alpha_dropout:
x = functional.alpha_dropout(input=x,
p=self.dropout_layers[l],
training=True,
inplace=False)
else:
x = functional.dropout(input=x,
p=self.dropout_layers[l],
training=True,
inplace=False)
return self.activation_output(self.fully_connected_layer(x))
def forward(self, data, mask):
x = data["data"]
x = z_normalize_tensor(x, self.input_mean, self.input_std)
x = F.selu(self.fc1(x.view(x.shape[0], -1)))
x = F.alpha_dropout(x, 0.3, self.training)
x = self.fc2(x)
return {"pred": F.log_softmax(x, dim=1)}
def forward(self):
a = torch.randn(8, 4)
b = torch.randn(8, 4, 4, 4)
c = torch.randn(8, 4, 4, 4, 4)
return len(
F.dropout(a),
F.dropout2d(b),
F.dropout3d(c),
F.alpha_dropout(a),
F.feature_alpha_dropout(c),
)
def forward(self, sentence):
embeds = self.word_embeddings(sentence)
shuffle = embeds.transpose(1, 2)
shuffle = F.dropout(shuffle, p=DROPOUT, training=self.training)
conv_1 = self.pool_1(self.selu_2(self.conv_1(shuffle)))
conv_1 = F.alpha_dropout(conv_1, p=DROPOUT, training=self.training)
conv_2 = self.pool_2(self.selu_3(self.conv_2(conv_1)))
conv_2 = F.alpha_dropout(conv_2, p=DROPOUT, training=self.training)
conv_3 = self.pool_3(self.selu_4(self.conv_3(conv_2)))
conv_3 = F.alpha_dropout(conv_3, p=DROPOUT, training=self.training)
global_pool = conv_3.mean(dim=-1)
# flat = self.selu_5(self.flat(global_pool))
# flat = F.alpha_dropout(flat, p=DROPOUT, training=self.training)
output_cnn = self.predict_cnn(global_pool)
return output_cnn
def forward(self, sentence):
hidden_gru = self.init_hidden_gru(sentence.size(0))
embeds = self.word_embeddings(sentence)
gru_input = F.alpha_dropout(embeds,
p=DROPOUT,
training=self.training)
gru_out, self.hidden_gru = self.gru(gru_input, hidden_gru)
concatenated_output = torch.cat(
[gru_out.select(1, WINDOW_SIZE - 1).contiguous()], dim=1)
concatenated_output = F.dropout(concatenated_output,
p=DROPOUT,
training=self.training)
output_gru = self.predict_lstm(concatenated_output)
return output_gru
def dropout(x,
dropout_rate=0,
training=False,
dropout_type='variational',
use_cuda=False):
"""
Args:
x (Tensor): (batch * len * input_size) or (any other shape)
"""
if dropout_type not in ('variational', 'simple', 'alpha'):
raise ValueError('Unknown dropout type = {}'.format(dropout_type))
if dropout_rate > 0:
# if x is (batch * len * input_size)
if dropout_type == 'variational' and len(x.size()) == 3:
return variational_dropout(x,
dropout_rate=dropout_rate,
training=training,
use_cuda=use_cuda)
elif dropout_type == 'simple':
return F.dropout(x, p=dropout_rate, training=training)
else:
return F.alpha_dropout(x, p=dropout_rate, training=training)
else:
return x
def test_alpha_dropout(self):
inp = torch.randn(16, 8, 64, 64, device='cuda', dtype=self.dtype)
output = F.alpha_dropout(inp, p=0.5, training=True, inplace=False)
def alpha_dropout(input, *args, **kwargs):
return _wrap_tensor(input, F.alpha_dropout(input.F, *args, **kwargs))
def forward(self, token_index_sequences, transitions):
buffers = self.embed(token_index_sequences)
buffers = tfunc.alpha_dropout(tfunc.selu(self.W_in(buffers)),
self.alph_dropout,
training=self.train_phase)
outputs0 = tfunc.log_softmax(self.W_out(buffers), 2).transpose(1, 0)
buffers = [
list(torch.split(b.squeeze(0), 1, 0))[::-1]
for b in torch.split(buffers, 1, 0)
]
transitions.transpose_(1, 0)
# The input comes in as a single tensor of word embeddings;
# I need it to be a list of stacks, one for each example in
# the batch, that we can pop from independently. The words in
# each example have already been reversed, so that they can
# be read from left to right by popping from the end of each
# list; they have also been prefixed with a null value.
# shape = (max_len, batch, embed_dims)
buffers = [
list(map(lambda vec_: torch.cat([vec_, vec_], 1), buf))
for buf in buffers
]
pad_embed = buffers[0][0]
stacks = [[pad_embed, pad_embed] for _ in buffers]
self.tracker.reset_state()
# TODO
# shape = (max_len, batch)
num_transitions = transitions.size(0)
outputs1 = list()
for i in range(num_transitions):
trans = transitions[i]
tracker_states = self.tracker(buffers, stacks)
lefts, rights, trackings = [], [], []
batch = list(zip(trans.data, buffers, stacks, tracker_states))
for bi in range(len(batch)):
transition, buf, stack, tracking = batch[bi]
if transition == self.SHIFT_SYMBOL: # shift
stack.append(buf.pop())
elif transition == self.REDUCE_SYMBOL: # reduce
rights.append(stack.pop())
lefts.append(stack.pop())
trackings.append(tracking)
# make sure tree are good
while len(stack) < 2:
stack.append(pad_embed)
if rights:
hc_list, hc_tensor = self.reduce(lefts, rights, trackings)
reduced = iter(hc_list)
for transition, stack in zip(trans.data, stacks):
if transition == 2:
stack.append(next(reduced))
outputs1.append(tfunc.log_softmax(self.W_out(hc_tensor[0]), 1))
# shape2 = (max_len, batch_size, output_dim)
# shape1 = (max_len, [num_reduce], output_dim)
return outputs0, outputs1
