def forward(self, x): for ii in range(0, len(self._convs)): #x = F.leaky_relu( self._convs[ii](x) ) #x = self._batchs2d[ii](x) #x = F.dropout2d( x, p=self._p_conv ) x = F.selu(self._convs[ii](x)) x = self._batchs2d[ii]( x) # seems to work better with batch norm ¯\_(ツ)_/¯ x = F.alpha_dropout(x, p=self._p_conv) if self._transfer: with torch.no_grad(): u = self._us v = self._vs for ii in range(0, len(self._n_convs)): w = self._convs[ii].weight self._n_convs[ii].set_weights(w) u = self._n_convs[ii](u) v = self._n_convs[ii](v) # 1st: just concatenate #x = torch.cat( ( x, u, v ), 1 ) # 2nd: attempt at using 1x1 convs on each channel of [x,u,v] images #n, c, h, w = x.shape #x_merged = torch.zeros( ( n, 2*c, h, w ), # dtype=x.dtype, # device=torch.device('cuda:0'), # requires_grad=True ) #print( 'x_merged shape ', x_merged.shape ) #for ii in range( 0, c ): # temp = torch.cat( ( x[:,ii:ii+1,:,:], u[:,ii:ii+1,:,:], v[:,ii:ii+1,:,:] ), 1 ) # print( 'temp shape ', temp.shape ) # print( 'ii ', ii ) # #x_merged[ :, 2*ii:2*ii+1, :, : ] = self._merge_convs[2*ii]( temp ) # #x_merged[ :, 2*ii+1:2*ii+2, :, : ] = self._merge_convs[2*ii+1]( temp ) # x_merged[ :, 2*ii:2*ii+2, :, : ] = self._merge_convs[ii]( temp ) # 3rd: concatenate and merge x = torch.cat((x, u, v), 1) x = self._merge_conv(x) x = x.view(-1, self._n) n_in = self._capacity - 1 for ii in range(0, n_in): #x = F.leaky_relu( self._fcs_in[ii](x) ) #if ( ii != n_in-1 ) : #x = self._batchs1d[ii](x) #x = F.dropout( x, p=self._p_fc ) x = F.selu(self._fcs_in[ii](x)) if (ii != n_in - 1): x = self._batchs1d[ii](x) x = F.alpha_dropout(x, p=self._p_fc) x = self._fcs_out[n_in](x) return x
def forward(self, x, y, z, w): x = F.alpha_dropout(x, training=False) y = F.alpha_dropout(y, training=False) z = F.alpha_dropout(z, p=0.6, training=False) w = F.alpha_dropout(w, p=0.1, training=False) x = F.relu(x) y = F.relu(y) z = F.relu(z) w = F.relu(w) return x, y, z, w
def forward(self, x): """ Feed forward algorithm :param x: Input :return: Configured neural network """ if self.params.get('initializer') is None: self.params.update( dict(initializer=np.random.choice(a=list(INITIALIZER.keys())))) set_initializer(x=x, **self.params) x = x.float() for l, layer in enumerate(self.hidden_layers): x = self.activation_functions[l](layer(x)) if self.use_alpha_dropout: x = functional.alpha_dropout(input=x, p=self.dropout_layers[l], training=True, inplace=False) else: x = functional.dropout(input=x, p=self.dropout_layers[l], training=True, inplace=False) return self.activation_output(self.fully_connected_layer(x))
def forward(self, data, mask): x = data["data"] x = z_normalize_tensor(x, self.input_mean, self.input_std) x = F.selu(self.fc1(x.view(x.shape[0], -1))) x = F.alpha_dropout(x, 0.3, self.training) x = self.fc2(x) return {"pred": F.log_softmax(x, dim=1)}
def forward(self): a = torch.randn(8, 4) b = torch.randn(8, 4, 4, 4) c = torch.randn(8, 4, 4, 4, 4) return len( F.dropout(a), F.dropout2d(b), F.dropout3d(c), F.alpha_dropout(a), F.feature_alpha_dropout(c), )
def forward(self, sentence): embeds = self.word_embeddings(sentence) shuffle = embeds.transpose(1, 2) shuffle = F.dropout(shuffle, p=DROPOUT, training=self.training) conv_1 = self.pool_1(self.selu_2(self.conv_1(shuffle))) conv_1 = F.alpha_dropout(conv_1, p=DROPOUT, training=self.training) conv_2 = self.pool_2(self.selu_3(self.conv_2(conv_1))) conv_2 = F.alpha_dropout(conv_2, p=DROPOUT, training=self.training) conv_3 = self.pool_3(self.selu_4(self.conv_3(conv_2))) conv_3 = F.alpha_dropout(conv_3, p=DROPOUT, training=self.training) global_pool = conv_3.mean(dim=-1) # flat = self.selu_5(self.flat(global_pool)) # flat = F.alpha_dropout(flat, p=DROPOUT, training=self.training) output_cnn = self.predict_cnn(global_pool) return output_cnn
def forward(self, sentence): hidden_gru = self.init_hidden_gru(sentence.size(0)) embeds = self.word_embeddings(sentence) gru_input = F.alpha_dropout(embeds, p=DROPOUT, training=self.training) gru_out, self.hidden_gru = self.gru(gru_input, hidden_gru) concatenated_output = torch.cat( [gru_out.select(1, WINDOW_SIZE - 1).contiguous()], dim=1) concatenated_output = F.dropout(concatenated_output, p=DROPOUT, training=self.training) output_gru = self.predict_lstm(concatenated_output) return output_gru
def dropout(x, dropout_rate=0, training=False, dropout_type='variational', use_cuda=False): """ Args: x (Tensor): (batch * len * input_size) or (any other shape) """ if dropout_type not in ('variational', 'simple', 'alpha'): raise ValueError('Unknown dropout type = {}'.format(dropout_type)) if dropout_rate > 0: # if x is (batch * len * input_size) if dropout_type == 'variational' and len(x.size()) == 3: return variational_dropout(x, dropout_rate=dropout_rate, training=training, use_cuda=use_cuda) elif dropout_type == 'simple': return F.dropout(x, p=dropout_rate, training=training) else: return F.alpha_dropout(x, p=dropout_rate, training=training) else: return x
def test_alpha_dropout(self): inp = torch.randn(16, 8, 64, 64, device='cuda', dtype=self.dtype) output = F.alpha_dropout(inp, p=0.5, training=True, inplace=False)
def alpha_dropout(input, *args, **kwargs): return _wrap_tensor(input, F.alpha_dropout(input.F, *args, **kwargs))
def forward(self, token_index_sequences, transitions): buffers = self.embed(token_index_sequences) buffers = tfunc.alpha_dropout(tfunc.selu(self.W_in(buffers)), self.alph_dropout, training=self.train_phase) outputs0 = tfunc.log_softmax(self.W_out(buffers), 2).transpose(1, 0) buffers = [ list(torch.split(b.squeeze(0), 1, 0))[::-1] for b in torch.split(buffers, 1, 0) ] transitions.transpose_(1, 0) # The input comes in as a single tensor of word embeddings; # I need it to be a list of stacks, one for each example in # the batch, that we can pop from independently. The words in # each example have already been reversed, so that they can # be read from left to right by popping from the end of each # list; they have also been prefixed with a null value. # shape = (max_len, batch, embed_dims) buffers = [ list(map(lambda vec_: torch.cat([vec_, vec_], 1), buf)) for buf in buffers ] pad_embed = buffers[0][0] stacks = [[pad_embed, pad_embed] for _ in buffers] self.tracker.reset_state() # TODO # shape = (max_len, batch) num_transitions = transitions.size(0) outputs1 = list() for i in range(num_transitions): trans = transitions[i] tracker_states = self.tracker(buffers, stacks) lefts, rights, trackings = [], [], [] batch = list(zip(trans.data, buffers, stacks, tracker_states)) for bi in range(len(batch)): transition, buf, stack, tracking = batch[bi] if transition == self.SHIFT_SYMBOL: # shift stack.append(buf.pop()) elif transition == self.REDUCE_SYMBOL: # reduce rights.append(stack.pop()) lefts.append(stack.pop()) trackings.append(tracking) # make sure tree are good while len(stack) < 2: stack.append(pad_embed) if rights: hc_list, hc_tensor = self.reduce(lefts, rights, trackings) reduced = iter(hc_list) for transition, stack in zip(trans.data, stacks): if transition == 2: stack.append(next(reduced)) outputs1.append(tfunc.log_softmax(self.W_out(hc_tensor[0]), 1)) # shape2 = (max_len, batch_size, output_dim) # shape1 = (max_len, [num_reduce], output_dim) return outputs0, outputs1