def forward(self, x): x = grad_reverse(x, self.beta) x = F.ReLU(self.fc_1_inst(x)) x = F.ReLU(nn.BatchNorm1d(self.fc_2_inst(x))) y = nn.Softmax(x) x = nn.LogSoftmax(x) return x, y
def forward(self,question): embeddings = self.glove(question) question_lengths = [len(s) for s in question] q_enc_hiddens = None pack_padded_Q = pack_padded_sequence(embeddings, question_lengths) packed_q_enc_hiddens, (last_q_hidden, last_q_cell) = self.question_encoder(pack_padded_Q) q_enc_hiddens, _ = pad_packed_sequence(packed_q_enc_hiddens) soft_decision = self.kb(q_enc_hiddens) decision = torch.functional.softmax(soft_decision) #log_softmax? knowledge_bit = torch.dot(self.kb.weights, decision) # now we have the decision we want to multiply it again by self.kb z1 = self.qa_l1(knowledge_bit) a1 = F.ReLU(z1) z2 = self.qa_l2(a1) a2 = F.ReLU(z2) z3 = self.qa_l3(a2) a3 = F.ReLU(z3) z4_start = self.qa_start(a3) start = torch.functional.softmax(z4_start) z4_end = self.qa_end(a3) end = torch.functional.softmax(z4_end) return start, end
def forward(self, image_batch, salient_batch, train=True): image_batch.requires_grad_(False) salient_batch.requires_grad_(False) image = self.avg_adp_pool(image_batch) image = self.image_alex(image) image = image.view(self.mini_batch_size, -1) salient = self.avg_adp_pool(salient_batch) salient = self.salient_alex(salient) salient = salient.view(self.mini_batch_size, -1) salient_raw = self.salient_max_pool(salient_batch) salient_raw = salient_raw.view(self.mini_batch_size, -1) to_linear = torch.cat((image, salient, salient_raw), dim=1) p = 0.3 out = F.dropout(to_linear, p, training=train) out = self.lin1(F.ReLU(out)) out = F.dropout(out, p, training=train) out = self.lin2(F.ReLU(out)) out = out.view(self.mini_batch_size, 1, 54, 96) return out
def forward(self, x): # Hidden layer with sigmoid activation x = F.ReLU(self.fc1(x)) # Output layer with softmax activation x = F.ReLU(self.fc2(x)) x = F.doftmax(self.fc3(x), dim=1) return x
def forward(self, x): # Pass the input tensor through L1, L2 layer functions and operations x = F.ReLU(self.L1(x)) x = F.ReLU(self.L2(x)) # L3 layer with softmax activation x = F.softmax(self.output(x), dim=1) return x
def encoder(self , x) : #Code for the encoder y = self.conv1(x) y = F.ReLU(y) y = self.conv2(y) y = F.ReLU(y) y = self.conv3(y) return self.residual_stack(y)
def forward(self, x): if self.downsample: y1 = self.depthconv_5x5(x) y1 = self.batchnorm(y1) y1 = self.conv1x1_downsample(y1) y1 = self.batchnorm(y1) # name=name + "/expand_bn5") y1 = F.relu(y1) x2 = x else: y1 = self.no_downsample_y1( x) # nn.Lambda(lambda z: z[:, 0:in_split2_channels, :, :])(x) x2 = self.no_downsample_x2( x) # nn.Lambda(lambda z: z[:, in_split2_channels:, :, :])(x) ''' if is_channels_first(): y1 = lambda x: x[:,0:in_split2_channels, :, :] #nn.Lambda(lambda z: z[:, 0:in_split2_channels, :, :])(x) x2 = lambda x: x[:, in_split2_channels:,:, :] #nn.Lambda(lambda z: z[:, in_split2_channels:, :, :])(x) else: y1 = lambda z: z[:, :, :, 0:in_split2_channels] #nn.Lambda(lambda z: z[:, :, :, 0:in_split2_channels])(x) x2 = lambda z: z[:, :, :, in_split2_channels:]#nn.Lambda(lambda z: z[:, :, :, in_split2_channels:])(x) ''' # name=name + "/compress_conv1") y2 = self.Conv1x1_1(x2) y2 = self.batchnorm(y2) # name=name + "/compress_bn1") y2 = F.ReLU(y2) y2 = self.depthwise_conv5x5_2(y2) # name=name + "/dw_conv2") y2 = self.batchnorm(y2) # name=name + "/dw_bn2") y2 = self.conv1x1_2(y2) y2 = self.batchnorm(y2) # name=name + "/expand_bn3") y2 = F.ReLU(y2) ''' if self.use_se: y2 = se_block( x=y2, channels=mid_channels, name=name + "/se") if self.use_residual and not self.downsample: y2 = y2 + x2 # nn.add([y2, x2], name=name + "/add") x = torch.cat([y1, y2], dim=1) #get_channel_axis()) # nn.concatenate([y1, y2], axis=get_channel_axis(), name=name + "/concat") x = channel_shuffle_lambda( channels=out_channels, groups=2, name=name + "/c_shuffle")(x) ''' return x
def __init__(self, features, feature_dim, embed_dim, adj_lists, aggregator, num_sample=10, base_model=None, gcn=False, cuda=False, feature_transform=False, activation=None, skip_connection=False, name=None): super(Encoder, self).__init__() self.name = name self.features = features self.feat_dim = feature_dim self.adj_lists = adj_lists self.aggregator = aggregator self.num_sample = num_sample self.act = nn.PReLU() if activation == 'prelu' else F.ReLU() self.skip_connection = skip_connection if base_model != None: self.base_model = base_model self.gcn = gcn self.embed_dim = embed_dim self.cuda = cuda self.aggregator.cuda = cuda dim = self.feat_dim if self.gcn else 2 * self.feat_dim if skip_connection: dim = dim if self.name == 'l2' else 2 * dim self.weight = nn.Parameter( torch.FloatTensor(embed_dim, dim)) self.weight_skip = nn.Parameter( torch.FloatTensor(embed_dim, dim)) init.xavier_uniform_(self.weight) init.xavier_uniform_(self.weight_skip)
def forward(self, inputs): embeds = torch.mean(self.embeddings(inputs), dim=0).view((1, -1)) out = self.linear1(embeds) out = F.ReLU(out) out = self.linear2(out) log_probs = F.log_softmax(out, dim=1) return log_probs
def forward(self, inputs): embeds = torch.mean(self.embeddings(inputs), dim=0).view((1, -1)) out = self.linear1(embeds) out = F.ReLU(out) out = self.linear2(out) log_probs = (1 - self.reg_factor) * F.log_softmax(out) \ - self.reg_factor * torch.sum(self.graph_weights, dim=0) return log_probs
def forward(self, input): """ Args: input (*) <torch.Tensor>: Returns: output (*) <torch.Tensor>: """ output = F.ReLU(input) return output
def __init__(self): super(CVAE, self).__init__() self.relu = nn.ReLU() self.sigmoid = nn.Sigmoid() # For Encoder self.fcE = nn.Linear(X_dim + y_dim, h_dim) self.fcE_mu = nn.Linear(h_dim, Z_dim) self.fcE_var = nn.Linear(h_dim, Z_dim) # For Decoder self.fcD1 = nn.Linear(Z_dim + y_dim, h_dim) self.fcD2 = nn.Linear(h_dim, X_dim)
def forward(self, x): x = grad_reverse(x, self.beta) x = nn.bn_image(F.ReLU(self.conv_image(x))) x = nn.MaxPool2d(x, (2, 2)) x = nn.BatchNorm1d(flatten(x)) # convert to 1024*W*H x 1. x = torch.transpose(x, 0, 1) x = self.fc_1_image(x) #x = self.fc_2_image(x) # 1 x lala vector y = nn.Softmax(x) x = nn.LogSoftmax(x) return x, y
def decoder(self , x) : #Code for the decoder y = self.conv4(x) y = self.residual_stack(y) y = self.conv_trans1(y) y = F.ReLU(y) y = self.conv_trans2(y) return y
def forward(self,x): print('Input: ',x.shape) x = self.conv1(x) x = F.relu(x) print('After conv 1: ',x.shape) x = F.MaxPool3d(kernel_size=(1,4,4),stride=(2,2,2)) x = self.conv2(x) x = F.ReLU(x) x = F.MaxPool3d(kernel_size=(1,4,4),stride=(2,2,2)) print('After conv 2: ',x.shape) x = self.conv3(x) x = F.ReLU() x = F.MaxPool3d(kernel_size=(1,5,5),stride=(2,2,2)) x = x.reshape(x.size(0),-1) print('After conv 3: ',x.shape) x = F.relu(self.fc1(x)) print('After full conv 1: ',x.shape) x = F.relu(self.fc2(x)) print('After full conv 2: ',x.shape) return x
def __init__(self, input_dim, output_dim, support=1, featureless=True, init='glorot_uniform', activation='linear', weights=None, W_regularizer=None, num_bases=-1, b_regularizer=None, bias=False, dropout=0., **kwargs): super(GraphConvolution, self).__init__() #self.init = initializations.get(init) #self.activation = activations.get(activation) if activation == "relu": self.activation = nn.ReLU() elif activation == "softmax": self.activation = nn.Softmax(dim=-1) else: self.activation = F.ReLU() self.input_dim = input_dim self.output_dim = output_dim # number of features per node self.support = support # filter support / number of weights self.featureless = featureless # use/ignore input features self.dropout = dropout self.w_regularizer = nn.L1Loss() assert support >= 1 #TODO self.bias = bias self.initial_weights = weights self.num_bases = num_bases # these will be defined during build() #self.input_dim = None if self.num_bases > 0: self.W = nn.Parameter(torch.empty(self.input_dim * self.num_bases, self.output_dim, dtype=torch.float32, device=device)) self.W_comp = nn.Parameter(torch.empty(self.support, self.num_bases, dtype=torch.float32, device=device)) nn.init.xavier_uniform_(self.W_comp) else: self.W = nn.Parameter(torch.empty(self.input_dim * self.support, self.output_dim, dtype=torch.float32, device=device)) nn.init.xavier_uniform_(self.W) if self.bias: self.b = nn.Parameter(torch.empty(self.output_dim, dtype=torch.float32, device=device)) nn.init.xavier_uniform_(self.b) self.dropout = nn.Dropout(dropout)
def gradient_descent_pytorch_using_nn(): batch_size, input_dim, hidden_layer_size, output_dim = 64, 1000, 100, 10 x = torch.randn(batch_size, input_dim) y = torch.randn(batch_size, output_dim) #ici on définit notre réseaud de neuronne. on voit ici une première couche linaire avec les input #un relu et une dernière couche lineair avec les outputs model = torch.nn.Sequential( nn.Linear(input_dim, hidden_layer_size), nn.ReLU(), nn.Linear(hidden_layer_size, output_dim), ) #on définit la loss on moyen de la fonction nn qui récupère directement la fonction et son traitement loss_fn = nn.MSELoss(reduction='sum') #on définit l'optimizer qui permet de rentre l'update des poids automatique dans un facon spécifique ici Adam mais possible SGD ou autre #optimizer_fn = torch.optim.Adam(model.parameters(), lr=learning_rate) for i in range(epochs): #ici nos input rentre dans le model y_pred = model(x) #on définit la fonction de loss avec le y_pred ainsi que les outputs loss = loss_fn(y_pred, y) #on met les gradients à 0 avant de faire le backward pass model.zero_grad() #calcule le gradient de la loss en fonction de tout les parametres possible du model. # à l'interne les paramètres de tous les modules ont le requires_grad=True # donc cette appel calculera le gradients de tout les paramètres du model loss.backward() #soit on fait manuellement comme précedement # mais la vu que notre model nous retour tout les paramètres on les récupères directement # grace à parametre() et on update les poids en fonctzion du learning rate with torch.no_grad(): for param in model.parameters(): param -= learning_rate * param.grad
def __init__(self, game, args): # game params self.board_x, self.board_y = game.getBoardSize() self.action_size = game.getActionSize() self.args = args super(netA, self).__init__() self.ngpu = ngpu self.fc3 = nn.Linear(512, self.action_size) self.fc4 = nn.Linear(512, 1) self.main = nn.Sequential( #Block 1: nn.Conv2d(1, args.num_channels, 3, stride=1, padding=1), nn.BatchNorm2d(args.num_channels), F.ReLU(), #Block 2: nn.Conv2d(args.num_channels, args.num_channels, 3, stride=1, padding=1), nn.BatchNorm2d(args.num_channels), F.ReLU(), #Block 3: nn.Conv2d(args.num_channels, args.num_channels, 3, stride=1), nn.BatchNorm2d(args.num_channels), F.ReLU(), #Block 4: nn.Conv2d(args.num_channels, args.num_channels, 3, stride=1), nn.BatchNorm2d(args.num_channels), F.ReLU(), #decode block 1: Flatten(), nn.Linear(args.num_channels*4*4, 1024), nn.BatchNorm2d(args.num_channels), F.ReLU(), F.dropout(p=self.args.dropout, training=self.training), #decode block 2: nn.Linear(1024, 512), nn.BatchNorm2d(512), F.ReLU(), F.dropout(p=self.args.dropout, training=self.training), # nn.Linear(512, self.action_size), nn.Linear(512, 1) )
def __call__(self, base_encoding, side_encoding, additional_encodings=[]): merged_encoding = self.base_net(base_encoding) + self.side_net(side_encoding) if self.dense: merged_encoding += sum([net(add_encoding) for net, add_encoding in zip(self.dense_side_nets, additional_encodings)]) return F.ReLU(merged_encoding)
def forward(self, x1, x2): feat1 = self.cnn(x1) feat2 = self.cnn(x2) return self.linear2( functional.ReLU(self.linear1(torch.cat((feat1, feat2), dim=-1))))
def forward(self, x): x = F.ReLU(self.h1(x)) x = F.ReLU(self.h2(x)) x = F.softmax(self.output, dim=1) return x
def forward(self, x): x = self.conv(x) x = F.ReLU(self.norm(x)) return x
def forward(self ,x) : for i in range(self.num_residual_layers) : x = self.layers[i](x) return F.ReLU(x) #Using functional relu as seperate layer is not needed
def __init__(self, in_channels, out_channels, **kwargs): super(BasicConv2d, self).__init__() self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs) self.bn = nn.BatchNorm2d(out_channels, eps=0.001) self.ReLU = F.ReLU()
def OneModule_loss(data, label, weight, bias): f = F.sigmoid(F.ReLU(weight.mm(data) + bias)) loss_sum = -(torch.log(f) * label).sum() return loss
def SRELU(x): return F.ReLU(x*1.4142)