def forward(self, x): x = grad_reverse(x, self.beta) x = F.ReLU(self.fc_1_inst(x)) x = F.ReLU(nn.BatchNorm1d(self.fc_2_inst(x))) y = nn.Softmax(x) x = nn.LogSoftmax(x) return x, y
def forward(self,question):
embeddings = self.glove(question)
question_lengths = [len(s) for s in question]
q_enc_hiddens = None
pack_padded_Q = pack_padded_sequence(embeddings, question_lengths)
packed_q_enc_hiddens, (last_q_hidden, last_q_cell) = self.question_encoder(pack_padded_Q)
q_enc_hiddens, _ = pad_packed_sequence(packed_q_enc_hiddens)
soft_decision = self.kb(q_enc_hiddens)
decision = torch.functional.softmax(soft_decision) #log_softmax?
knowledge_bit = torch.dot(self.kb.weights, decision)
# now we have the decision we want to multiply it again by self.kb
z1 = self.qa_l1(knowledge_bit)
a1 = F.ReLU(z1)
z2 = self.qa_l2(a1)
a2 = F.ReLU(z2)
z3 = self.qa_l3(a2)
a3 = F.ReLU(z3)
z4_start = self.qa_start(a3)
start = torch.functional.softmax(z4_start)
z4_end = self.qa_end(a3)
end = torch.functional.softmax(z4_end)
return start, end
def forward(self, image_batch, salient_batch, train=True):
image_batch.requires_grad_(False)
salient_batch.requires_grad_(False)
image = self.avg_adp_pool(image_batch)
image = self.image_alex(image)
image = image.view(self.mini_batch_size, -1)
salient = self.avg_adp_pool(salient_batch)
salient = self.salient_alex(salient)
salient = salient.view(self.mini_batch_size, -1)
salient_raw = self.salient_max_pool(salient_batch)
salient_raw = salient_raw.view(self.mini_batch_size, -1)
to_linear = torch.cat((image, salient, salient_raw), dim=1)
p = 0.3
out = F.dropout(to_linear, p, training=train)
out = self.lin1(F.ReLU(out))
out = F.dropout(out, p, training=train)
out = self.lin2(F.ReLU(out))
out = out.view(self.mini_batch_size, 1, 54, 96)
return out
def forward(self, x):
# Hidden layer with sigmoid activation
x = F.ReLU(self.fc1(x))
# Output layer with softmax activation
x = F.ReLU(self.fc2(x))
x = F.doftmax(self.fc3(x), dim=1)
return x
def forward(self, x):
# Pass the input tensor through L1, L2 layer functions and operations
x = F.ReLU(self.L1(x))
x = F.ReLU(self.L2(x))
# L3 layer with softmax activation
x = F.softmax(self.output(x), dim=1)
return x
def encoder(self , x) : #Code for the encoder y = self.conv1(x) y = F.ReLU(y) y = self.conv2(y) y = F.ReLU(y) y = self.conv3(y) return self.residual_stack(y)
def forward(self, x):
if self.downsample:
y1 = self.depthconv_5x5(x)
y1 = self.batchnorm(y1)
y1 = self.conv1x1_downsample(y1)
y1 = self.batchnorm(y1) # name=name + "/expand_bn5")
y1 = F.relu(y1)
x2 = x
else:
y1 = self.no_downsample_y1(
x) # nn.Lambda(lambda z: z[:, 0:in_split2_channels, :, :])(x)
x2 = self.no_downsample_x2(
x) # nn.Lambda(lambda z: z[:, in_split2_channels:, :, :])(x)
'''
if is_channels_first():
y1 = lambda x: x[:,0:in_split2_channels, :, :] #nn.Lambda(lambda z: z[:, 0:in_split2_channels, :, :])(x)
x2 = lambda x: x[:, in_split2_channels:,:, :] #nn.Lambda(lambda z: z[:, in_split2_channels:, :, :])(x)
else:
y1 = lambda z: z[:, :, :, 0:in_split2_channels] #nn.Lambda(lambda z: z[:, :, :, 0:in_split2_channels])(x)
x2 = lambda z: z[:, :, :, in_split2_channels:]#nn.Lambda(lambda z: z[:, :, :, in_split2_channels:])(x)
'''
# name=name + "/compress_conv1")
y2 = self.Conv1x1_1(x2)
y2 = self.batchnorm(y2) # name=name + "/compress_bn1")
y2 = F.ReLU(y2)
y2 = self.depthwise_conv5x5_2(y2) # name=name + "/dw_conv2")
y2 = self.batchnorm(y2) # name=name + "/dw_bn2")
y2 = self.conv1x1_2(y2)
y2 = self.batchnorm(y2) # name=name + "/expand_bn3")
y2 = F.ReLU(y2)
'''
if self.use_se:
y2 = se_block(
x=y2,
channels=mid_channels,
name=name + "/se")
if self.use_residual and not self.downsample:
y2 = y2 + x2 # nn.add([y2, x2], name=name + "/add")
x = torch.cat([y1, y2], dim=1) #get_channel_axis()) # nn.concatenate([y1, y2], axis=get_channel_axis(), name=name + "/concat")
x = channel_shuffle_lambda(
channels=out_channels,
groups=2,
name=name + "/c_shuffle")(x)
'''
return x
def __init__(self, features, feature_dim, embed_dim, adj_lists, aggregator,
num_sample=10, base_model=None, gcn=False, cuda=False,
feature_transform=False, activation=None, skip_connection=False, name=None):
super(Encoder, self).__init__()
self.name = name
self.features = features
self.feat_dim = feature_dim
self.adj_lists = adj_lists
self.aggregator = aggregator
self.num_sample = num_sample
self.act = nn.PReLU() if activation == 'prelu' else F.ReLU()
self.skip_connection = skip_connection
if base_model != None:
self.base_model = base_model
self.gcn = gcn
self.embed_dim = embed_dim
self.cuda = cuda
self.aggregator.cuda = cuda
dim = self.feat_dim if self.gcn else 2 * self.feat_dim
if skip_connection:
dim = dim if self.name == 'l2' else 2 * dim
self.weight = nn.Parameter(
torch.FloatTensor(embed_dim, dim))
self.weight_skip = nn.Parameter(
torch.FloatTensor(embed_dim, dim))
init.xavier_uniform_(self.weight)
init.xavier_uniform_(self.weight_skip)
def forward(self, inputs):
embeds = torch.mean(self.embeddings(inputs), dim=0).view((1, -1))
out = self.linear1(embeds)
out = F.ReLU(out)
out = self.linear2(out)
log_probs = F.log_softmax(out, dim=1)
return log_probs
def forward(self, inputs):
embeds = torch.mean(self.embeddings(inputs), dim=0).view((1, -1))
out = self.linear1(embeds)
out = F.ReLU(out)
out = self.linear2(out)
log_probs = (1 - self.reg_factor) * F.log_softmax(out) \
- self.reg_factor * torch.sum(self.graph_weights, dim=0)
return log_probs
def forward(self, input):
"""
Args:
input (*) <torch.Tensor>:
Returns:
output (*) <torch.Tensor>:
"""
output = F.ReLU(input)
return output
def __init__(self):
super(CVAE, self).__init__()
self.relu = nn.ReLU()
self.sigmoid = nn.Sigmoid()
# For Encoder
self.fcE = nn.Linear(X_dim + y_dim, h_dim)
self.fcE_mu = nn.Linear(h_dim, Z_dim)
self.fcE_var = nn.Linear(h_dim, Z_dim)
# For Decoder
self.fcD1 = nn.Linear(Z_dim + y_dim, h_dim)
self.fcD2 = nn.Linear(h_dim, X_dim)
def forward(self, x): x = grad_reverse(x, self.beta) x = nn.bn_image(F.ReLU(self.conv_image(x))) x = nn.MaxPool2d(x, (2, 2)) x = nn.BatchNorm1d(flatten(x)) # convert to 1024*W*H x 1. x = torch.transpose(x, 0, 1) x = self.fc_1_image(x) #x = self.fc_2_image(x) # 1 x lala vector y = nn.Softmax(x) x = nn.LogSoftmax(x) return x, y
def decoder(self , x) : #Code for the decoder y = self.conv4(x) y = self.residual_stack(y) y = self.conv_trans1(y) y = F.ReLU(y) y = self.conv_trans2(y) return y
def forward(self,x):
print('Input: ',x.shape)
x = self.conv1(x)
x = F.relu(x)
print('After conv 1: ',x.shape)
x = F.MaxPool3d(kernel_size=(1,4,4),stride=(2,2,2))
x = self.conv2(x)
x = F.ReLU(x)
x = F.MaxPool3d(kernel_size=(1,4,4),stride=(2,2,2))
print('After conv 2: ',x.shape)
x = self.conv3(x)
x = F.ReLU()
x = F.MaxPool3d(kernel_size=(1,5,5),stride=(2,2,2))
x = x.reshape(x.size(0),-1)
print('After conv 3: ',x.shape)
x = F.relu(self.fc1(x))
print('After full conv 1: ',x.shape)
x = F.relu(self.fc2(x))
print('After full conv 2: ',x.shape)
return x
def __init__(self, input_dim, output_dim, support=1, featureless=True,
init='glorot_uniform', activation='linear',
weights=None, W_regularizer=None, num_bases=-1,
b_regularizer=None, bias=False, dropout=0., **kwargs):
super(GraphConvolution, self).__init__()
#self.init = initializations.get(init)
#self.activation = activations.get(activation)
if activation == "relu":
self.activation = nn.ReLU()
elif activation == "softmax":
self.activation = nn.Softmax(dim=-1)
else:
self.activation = F.ReLU()
self.input_dim = input_dim
self.output_dim = output_dim # number of features per node
self.support = support # filter support / number of weights
self.featureless = featureless # use/ignore input features
self.dropout = dropout
self.w_regularizer = nn.L1Loss()
assert support >= 1
#TODO
self.bias = bias
self.initial_weights = weights
self.num_bases = num_bases
# these will be defined during build()
#self.input_dim = None
if self.num_bases > 0:
self.W = nn.Parameter(torch.empty(self.input_dim * self.num_bases, self.output_dim, dtype=torch.float32, device=device))
self.W_comp = nn.Parameter(torch.empty(self.support, self.num_bases, dtype=torch.float32, device=device))
nn.init.xavier_uniform_(self.W_comp)
else:
self.W = nn.Parameter(torch.empty(self.input_dim * self.support, self.output_dim, dtype=torch.float32, device=device))
nn.init.xavier_uniform_(self.W)
if self.bias:
self.b = nn.Parameter(torch.empty(self.output_dim, dtype=torch.float32, device=device))
nn.init.xavier_uniform_(self.b)
self.dropout = nn.Dropout(dropout)
def gradient_descent_pytorch_using_nn():
batch_size, input_dim, hidden_layer_size, output_dim = 64, 1000, 100, 10
x = torch.randn(batch_size, input_dim)
y = torch.randn(batch_size, output_dim)
#ici on définit notre réseaud de neuronne. on voit ici une première couche linaire avec les input
#un relu et une dernière couche lineair avec les outputs
model = torch.nn.Sequential(
nn.Linear(input_dim, hidden_layer_size),
nn.ReLU(),
nn.Linear(hidden_layer_size, output_dim),
)
#on définit la loss on moyen de la fonction nn qui récupère directement la fonction et son traitement
loss_fn = nn.MSELoss(reduction='sum')
#on définit l'optimizer qui permet de rentre l'update des poids automatique dans un facon spécifique ici Adam mais possible SGD ou autre
#optimizer_fn = torch.optim.Adam(model.parameters(), lr=learning_rate)
for i in range(epochs):
#ici nos input rentre dans le model
y_pred = model(x)
#on définit la fonction de loss avec le y_pred ainsi que les outputs
loss = loss_fn(y_pred, y)
#on met les gradients à 0 avant de faire le backward pass
model.zero_grad()
#calcule le gradient de la loss en fonction de tout les parametres possible du model.
# à l'interne les paramètres de tous les modules ont le requires_grad=True
# donc cette appel calculera le gradients de tout les paramètres du model
loss.backward()
#soit on fait manuellement comme précedement
# mais la vu que notre model nous retour tout les paramètres on les récupères directement
# grace à parametre() et on update les poids en fonctzion du learning rate
with torch.no_grad():
for param in model.parameters():
param -= learning_rate * param.grad
def __init__(self, game, args):
# game params
self.board_x, self.board_y = game.getBoardSize()
self.action_size = game.getActionSize()
self.args = args
super(netA, self).__init__()
self.ngpu = ngpu
self.fc3 = nn.Linear(512, self.action_size)
self.fc4 = nn.Linear(512, 1)
self.main = nn.Sequential(
#Block 1:
nn.Conv2d(1, args.num_channels, 3, stride=1, padding=1),
nn.BatchNorm2d(args.num_channels),
F.ReLU(),
#Block 2:
nn.Conv2d(args.num_channels, args.num_channels, 3, stride=1, padding=1),
nn.BatchNorm2d(args.num_channels),
F.ReLU(),
#Block 3:
nn.Conv2d(args.num_channels, args.num_channels, 3, stride=1),
nn.BatchNorm2d(args.num_channels),
F.ReLU(),
#Block 4:
nn.Conv2d(args.num_channels, args.num_channels, 3, stride=1),
nn.BatchNorm2d(args.num_channels),
F.ReLU(),
#decode block 1:
Flatten(),
nn.Linear(args.num_channels*4*4, 1024),
nn.BatchNorm2d(args.num_channels),
F.ReLU(),
F.dropout(p=self.args.dropout, training=self.training),
#decode block 2:
nn.Linear(1024, 512),
nn.BatchNorm2d(512),
F.ReLU(),
F.dropout(p=self.args.dropout, training=self.training),
#
nn.Linear(512, self.action_size),
nn.Linear(512, 1)
)
def __call__(self, base_encoding, side_encoding, additional_encodings=[]):
merged_encoding = self.base_net(base_encoding) + self.side_net(side_encoding)
if self.dense:
merged_encoding += sum([net(add_encoding) for net, add_encoding in zip(self.dense_side_nets, additional_encodings)])
return F.ReLU(merged_encoding)
def forward(self, x1, x2):
feat1 = self.cnn(x1)
feat2 = self.cnn(x2)
return self.linear2(
functional.ReLU(self.linear1(torch.cat((feat1, feat2), dim=-1))))
def forward(self, x):
x = F.ReLU(self.h1(x))
x = F.ReLU(self.h2(x))
x = F.softmax(self.output, dim=1)
return x
def forward(self, x):
x = self.conv(x)
x = F.ReLU(self.norm(x))
return x
def forward(self ,x) : for i in range(self.num_residual_layers) : x = self.layers[i](x) return F.ReLU(x) #Using functional relu as seperate layer is not needed
def __init__(self, in_channels, out_channels, **kwargs):
super(BasicConv2d, self).__init__()
self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)
self.bn = nn.BatchNorm2d(out_channels, eps=0.001)
self.ReLU = F.ReLU()
def OneModule_loss(data, label, weight, bias):
f = F.sigmoid(F.ReLU(weight.mm(data) + bias))
loss_sum = -(torch.log(f) * label).sum()
return loss
def SRELU(x):
return F.ReLU(x*1.4142)
