def forward(self, sentence):
# print(sentence) [torch.LongTensor of size 47x64]
x = self.word_embeddings(sentence) # [torch.FloatTensor of size 47x64x100]
x = torch.transpose(x, 0, 1)
# print(x) # [torch.FloatTensor of size 32x43x300]
x = x.unsqueeze(1)
# x = F.relu(self.convl3(x).squeeze(3))
x = [F.relu(conv(x)).squeeze(3) for conv in self.convsl]
# print(x) # [torch.FloatTensor of size 32x200x41] 40 39 38 37
# print(type(x))
# a = torch.cat((a1.data, a2.data), 2)
x = torch.cat(x, 2)
# print(x) # [torch.FloatTensor of size 32x200x195]
x = torch.transpose(x, 1, 2)
embeds = torch.transpose(x, 0, 1)
# print(embeds) # [torch.FloatTensor of size 195x32x200]
# embeds = self.word_embeddings()
# x = embeds.view(len(sentence), self.batch_size, -1) # torch.Size([43, 64, 300])
lstm_out, self.hidden = self.lstm(embeds, self.hidden)
lstm_out = torch.transpose(lstm_out, 0, 1)
lstm_out = torch.transpose(lstm_out, 1, 2)
# lstm_out = F.tanh(lstm_out)
lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2))
# print(lstm_out.size())
lstm_out = lstm_out.squeeze(2)
lstm_out = self.dropout(lstm_out)
y = self.hidden2label(lstm_out)
# log_probs = F.log_softmax(y)
log_probs = y
return log_probs
def get_paf_and_heatmap(model, img_raw, scale_search, param_stride=8, box_size=368):
multiplier = [scale * box_size / img_raw.shape[0] for scale in scale_search]
heatmap_avg = torch.zeros((len(multiplier), 19, img_raw.shape[0], img_raw.shape[1])).cuda()
paf_avg = torch.zeros((len(multiplier), 38, img_raw.shape[0], img_raw.shape[1])).cuda()
for i, scale in enumerate(multiplier):
img_test = cv2.resize(img_raw, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
img_test_pad, pad = pad_right_down_corner(img_test, param_stride, param_stride)
img_test_pad = np.transpose(np.float32(img_test_pad[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
feed = Variable(torch.from_numpy(img_test_pad)).cuda()
output1, output2 = model(feed)
print(output1.size())
print(output2.size())
heatmap = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output2)
paf = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output1)
heatmap_avg[i] = heatmap[0].data
paf_avg[i] = paf[0].data
heatmap_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda()
heatmap_avg = heatmap_avg.cpu().numpy()
paf_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(paf_avg, 0)), 0, 1), 1, 2).cuda()
paf_avg = paf_avg.cpu().numpy()
return paf_avg, heatmap_avg
def forward(self, x):
x = self.embed(x)
x = self.dropout(x)
# x = x.view(len(x), x.size(1), -1)
# x = embed.view(len(x), embed.size(1), -1)
bilstm_out, self.hidden = self.bilstm(x, self.hidden)
bilstm_out = torch.transpose(bilstm_out, 0, 1)
bilstm_out = torch.transpose(bilstm_out, 1, 2)
# bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2)).squeeze(2)
bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2))
bilstm_out = bilstm_out.squeeze(2)
hidden2lable = self.hidden2label1(F.tanh(bilstm_out))
gate_layer = F.sigmoid(self.gate_layer(bilstm_out))
# calculate highway layer values
gate_hidden_layer = torch.mul(hidden2lable, gate_layer)
# if write like follow ,can run,but not equal the HighWay NetWorks formula
# gate_input = torch.mul((1 - gate_layer), hidden2lable)
gate_input = torch.mul((1 - gate_layer), bilstm_out)
highway_output = torch.add(gate_hidden_layer, gate_input)
logit = self.logit_layer(highway_output)
return logit
def score(self, hidden, encoder_output):
if self.method == 'dot':
# hidden is 1 by 256
# encoder_output is 22 by 256
encoder_output = torch.transpose(encoder_output, 0, 1)
# encoder_output is 256 by 22
energy = torch.matmul(hidden, encoder_output)
return energy
elif self.method == 'general':
# hidden is 1 by 256
# encoder_output is 256 by 22
# encoder_output = torch.transpose(encoder_output, 0, 1)
hidden = hidden.view(1, -1)
a = self.attn(encoder_output)
a = torch.transpose(a, 0, 1)
energy = torch.matmul(hidden, a)
return energy
elif self.method == 'concat':
len_encoder_output = encoder_output.size()[1]
# hidden is 1 by 256
# encoder_output is 256 by 22
hidden = torch.transpose(hidden, 0, 1)
# hidden is 256 by 1
hidden = hidden.repeat(hidden_size, len_encoder_output)
# hidden is 256 by 22
concat = torch.cat((hidden, encoder_output), dim=0)
# concat is 512 by 22
# self.attn(concat) --> 256 by 22
energy = torch.matmul(self.v, F.tanh(self.attn(concat)))
return energy
def forward(self, sentence):
# print(sentence) # [torch.LongTensor of size 42x64]
x = self.word_embeddings(sentence)
x = self.dropout_embed(x)
# print(embeds.size()) # torch.Size([42, 64, 100])
# x = embeds.view(len(sentence), self.batch_size, -1)
print(x.size()) # torch.Size([42, 64, 100])
print(self.hidden)
lstm_out, self.hidden = self.bnlstm(x, self.hidden) # lstm_out 10*5*50 hidden 1*5*50 *2
# print(lstm_out)
# lstm_out = [F.max_pool1d(i, len(lstm_out)).unsqueeze(2) for i in lstm_out]
lstm_out = torch.transpose(lstm_out, 0, 1)
lstm_out = torch.transpose(lstm_out, 1, 2)
lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2))
# print(lstm_out.size())
lstm_out = lstm_out.squeeze(2)
# y = self.hidden2label(lstm_out)
#lstm_out = torch.cat(lstm_out, 1)
# lstm_out = self.dropout(lstm_out)
# lstm_out = lstm_out.view(len(sentence), -1)
y = self.hidden2label(F.tanh(lstm_out))
# log_probs = F.log_softmax(y)
log_probs = y
return log_probs
def forward(self, x):
'''
:param x: batch * input_len * in_channels
:return: batch * output_len * out_channels
'''
out= F.relu(self.maxPool(self.conv1(torch.transpose(x, 1, 2))))
#print('out: ', out.size())
out= F.relu(self.maxPool(self.conv2(out)))
out= torch.transpose(F.relu(self.maxPool(self.conv3(out))), 1, 2)
return out
def forward(self, X, X_mask):
#X: [m, Tx] m = batch size, Tx = word count
#print(X.size(), type(X))
m = X.size()[0]
Tx = X.size()[1]
#embedding layer
X = self.embedding(X)
#X: [m, Tx, embedding_dim] m = batch size, Tx = word count
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, self.embedding_dim])
#conv layer
#transpose
X = torch.transpose(X, 1, 2)
#print(X.size(), type(X.data))
X = self.conv(X)
#print(X.size(), type(X.data))
#transpose back
X = torch.transpose(X, 1, 2)
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, 256])
#maxpool layer
#transpose
X = torch.transpose(X, 1, 2)
X = self.maxpool(X)
#print(X.size(), type(X.data))
#remove dimension
X = X.squeeze()
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, 256])
#linear
X = self.linear(X)
#X: [m, 1]
#print(X.size(), type(X))
if self.num_classes == 2:
assert X.size() == torch.Size([m, 1])
else:
assert X.size() == torch.Size([m, self.num_classes])
if self.num_classes == 2:
X = torch.squeeze(X)
X = self.sigmoid(X)
#X: [m]
#print(X.size(), type(X))
assert X.size() == torch.Size([m])
return X
else:
return F.softmax(X)
def forward(self, x):
embed = self.embed(x)
x = embed.view(len(x), embed.size(1), -1)
bilstm_out, self.hidden = self.bilstm(x, self.hidden)
bilstm_out = torch.transpose(bilstm_out, 0, 1)
bilstm_out = torch.transpose(bilstm_out, 1, 2)
bilstm_out = F.tanh(bilstm_out)
bilstm_out = F.max_pool1d(bilstm_out, bilstm_out.size(2)).squeeze(2)
y = self.hidden2label1(bilstm_out)
y = self.hidden2label2(y)
logit = y
return logit
def myMatrixDivVector(matrix, vector):
"""
matrix(N,M) / vector(N) = matrix(N,M)
for each i,j:
matrix_result[i][j] = matrix_source[i][j] / vector[i]
"""
matrix1 = torch.transpose(matrix, 0, 1)
print("matrix transpose:", matrix1)
for i, nm in enumerate(matrix1):
matrix1[i] = nm / vector
print("matrix after division:", matrix1)
matrix = torch.transpose(matrix1, 0, 1)
print("matrix (final result):", matrix)
return matrix
def CORAL_loss(source, target):
d = source.data.shape[1]
# source covariance
xm = torch.mean(source, 1, keepdim=True) - source
xc = torch.matmul(torch.transpose(xm, 0, 1), xm)
# target covariance
xmt = torch.mean(target, 1, keepdim=True) - target
xct = torch.matmul(torch.transpose(xmt, 0, 1), xmt)
# frobenius norm between source and target
loss = (xc - xct).pow(2).sum().sqrt()
loss = loss/(4*d*d)
return loss
def forward(self, input):
embed = self.embed(input)
input = embed.view(len(input), embed.size(1), -1)
# lstm
lstm_out, hidden = self.gru(input, self.hidden)
lstm_out = torch.transpose(lstm_out, 0, 1)
lstm_out = torch.transpose(lstm_out, 1, 2)
# pooling
lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2)).squeeze(2)
lstm_out = F.tanh(lstm_out)
# linear
y = self.hidden2label(lstm_out)
logit = y
return logit
def max_singular_value(W, u=None, Ip=1):
"""
power iteration for weight parameter
"""
#xp = W.data
if u is None:
u = torch.FloatTensor(1, W.size(0)).normal_(0, 1).cuda()
_u = u
for _ in range(Ip):
#print(_u.size(), W.size())
_v = _l2normalize(torch.matmul(_u, W.data), eps=1e-12)
_u = _l2normalize(torch.matmul(_v, torch.transpose(W.data, 0, 1)), eps=1e-12)
sigma = torch.matmul(torch.matmul(_v, torch.transpose(W.data, 0, 1)), torch.transpose(_u, 0, 1))
return sigma, _v
def _get_lstm_features(self, names, lengths):
self.hidden = self.init_hidden(names.size(-1))
embeds = self.char_embeds(names) # Figure 4
packed_input = pack_padded_sequence(embeds, lengths) # Figure 5
packed_output, (ht, ct) = self.lstm(packed_input, self.hidden) # Figure 6
lstm_out, _ = pad_packed_sequence(packed_output) # Figure 7
lstm_out = torch.transpose(lstm_out, 0, 1)
lstm_out = torch.transpose(lstm_out, 1, 2)
lstm_out = F.tanh(lstm_out) # Figure 8
lstm_out, indices = F.max_pool1d(lstm_out, lstm_out.size(2), return_indices=True) # Figure 9
lstm_out = lstm_out.squeeze(2) #对维度的修正，使其符合输入格式
lstm_out = F.tanh(lstm_out)
lstm_feats = self.fully_connected_layer(lstm_out)
output = self.softmax(lstm_feats) # Figure 10
return output
def forward(self, x):
embed = self.embed(x)
embed = self.dropout_embed(embed)
x = embed.view(len(x), embed.size(1), -1)
# lstm
lstm_out, self.hidden = self.lstm(x, self.hidden)
lstm_out = torch.transpose(lstm_out, 0, 1)
lstm_out = torch.transpose(lstm_out, 1, 2)
# pooling
lstm_out = F.tanh(lstm_out)
lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2)).squeeze(2)
lstm_out = F.tanh(lstm_out)
# linear
logit = self.hidden2label(lstm_out)
return logit
def forward(self, question,length):
length = list(length.data.cpu().numpy())
emb = self.drop(self.encoder(question))
emb = self.tanh(emb)
hidden = self.init_hidden(len(length))
seqs = trnn.pack_padded_sequence(emb, length, batch_first=True)
seqs, hidden = self.rnn(seqs, hidden)
h,_ = trnn.pad_packed_sequence(seqs, batch_first=True)
#attention
weights = self.softmax(self.att2(torch.transpose(h, 1, 2)).squeeze(1)).unsqueeze(-1)
weights = weights.expand_as(h)
bilstmout = torch.sum(h*weights, 1).squeeze(1)
#bilstmout = torch.cat([hidden[0][0],hidden[0][1]],-1)
fc1fea = self.fc1(bilstmout)
return fc1fea
def forward(self, input):
embed = self.embed(input)
embed = self.dropout(embed) # add this reduce the acc
input = embed.view(len(input), embed.size(1), -1)
# gru
gru_out, hidden = self.bigru(input, self.hidden)
gru_out = torch.transpose(gru_out, 0, 1)
gru_out = torch.transpose(gru_out, 1, 2)
# pooling
# gru_out = F.tanh(gru_out)
gru_out = F.max_pool1d(gru_out, gru_out.size(2)).squeeze(2)
gru_out = F.tanh(gru_out)
# linear
y = self.hidden2label(gru_out)
logit = y
return logit
def forward(self, inputs):
'''inputs: batch_size, num_tokens, 50
return: batch_size, num_tokens + 2, output_dim'''
mask = ((inputs > 0).long().sum(dim=-1) > 0).long()
inputs_with_boundary, mask_with_boundary = _add_sentence_boundary_token_ids(inputs, mask,
self._beginning_of_sentence,
self._end_of_sentence)
max_chars_per_token = self._options["char_cnn"]["max_characters_per_token"]
character_embedding = torch.nn.functional.embedding(inputs_with_boundary.view(-1, max_chars_per_token),
self._char_embedding_weights)
assert self._options["char_cnn"]["activation"] == "relu"
# shape after transpose: (batch_size * (num_tokens + 2), output_dim, max_chars_per_token)
character_embedding = torch.transpose(character_embedding, 1, 2)
convs = []
for i in range(len(self._convolutions)):
conv = getattr(self, "char_conv_%d" % i)
convolved = conv(character_embedding)
# for each width, (batch_size * (num_tokens + 2), n_filters)
convolved, _ = torch.max(convolved, dim=-1)
convolved = torch.nn.functional.relu(convolved)
convs.append(convolved)
token_embedding = torch.cat(convs, dim=-1)
token_embedding = self._highways(token_embedding)
token_embedding = self._projection(token_embedding)
batch_size, sequence_length_with_boundary, _ = inputs_with_boundary.size()
return {"mask": mask_with_boundary,
"token_embedding": token_embedding.view(batch_size, sequence_length_with_boundary, -1)}
def forward(self, tokens: torch.Tensor, mask: torch.Tensor): # pylint: disable=arguments-differ
if mask is not None:
tokens = tokens * mask.unsqueeze(-1).float()
# Our input is expected to have shape `(batch_size, num_tokens, embedding_dim)`. The
# convolution layers expect input of shape `(batch_size, in_channels, sequence_length)`,
# where the conv layer `in_channels` is our `embedding_dim`. We thus need to transpose the
# tensor first.
tokens = torch.transpose(tokens, 1, 2)
# Each convolution layer returns output of size `(batch_size, num_filters, pool_length)`,
# where `pool_length = num_tokens - ngram_size + 1`. We then do an activation function,
# then do max pooling over each filter for the whole input sequence. Because our max
# pooling is simple, we just use `torch.max`. The resultant tensor of has shape
# `(batch_size, num_conv_layers * num_filters)`, which then gets projected using the
# projection layer, if requested.
filter_outputs = []
for i in range(len(self._convolution_layers)):
convolution_layer = getattr(self, 'conv_layer_{}'.format(i))
filter_outputs.append(
self._activation(convolution_layer(tokens)).max(dim=2)[0]
)
# Now we have a list of `num_conv_layers` tensors of shape `(batch_size, num_filters)`.
# Concatenating them gives us a tensor of shape `(batch_size, num_filters * num_conv_layers)`.
maxpool_output = torch.cat(filter_outputs, dim=1) if len(filter_outputs) > 1 else filter_outputs[0]
if self.projection_layer:
result = self.projection_layer(maxpool_output)
else:
result = maxpool_output
return result
def forward(self, x):
x_no_static = self.embed_no_static(x)
# x_no_static = self.dropout(x_no_static)
x_static = self.embed_static(x)
# fix the embedding
x_static = Variable(x_static.data)
# x_static = self.dropout(x_static)
x = torch.stack([x_static, x_no_static], 1)
one_layer = x # (N,W,D) # torch.Size([64, 43, 300])
# print("one_layer {}".format(one_layer.size()))
# one_layer = self.dropout(one_layer)
# one_layer = one_layer.unsqueeze(1) # (N,Ci,W,D) # torch.Size([64, 1, 43, 300])
# one layer
one_layer = [torch.transpose(F.relu(conv(one_layer)).squeeze(3), 1, 2).unsqueeze(1) for conv in self.convs1] # torch.Size([64, 100, 36])
# one_layer = [F.relu(conv(one_layer)).squeeze(3).unsqueeze(1) for conv in self.convs1] # torch.Size([64, 100, 36])
# print(one_layer[0].size())
# print(one_layer[1].size())
# two layer
two_layer = [F.relu(conv(one_layer)).squeeze(3) for (conv, one_layer) in zip(self.convs2, one_layer)]
# print("two_layer {}".format(two_layer[0].size()))
# print("two_layer {}".format(two_layer[1].size()))
# pooling
output = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in two_layer] # torch.Size([64, 100]) torch.Size([64, 100])
output = torch.cat(output, 1) # torch.Size([64, 300])
# dropout
output = self.dropout(output)
# linear
output = self.fc1(output)
logit = self.fc2(F.relu(output))
return logit
def forward(self, x):
# print(x) # <class 'torch.autograd.variable.Variable'> [torch.LongTensor of size 64x35]
# x_size = x.data.size(1)
# print(x_size)
x = self.embed(x)
x = x.unsqueeze(1) # (N,Ci,W,D) 在索引1处增加一维
# print(x) # [torch.FloatTensor of size 64x1x35x128]
# x = Variable(torch.transpose(x.data, 0, 1))
# x = self.bn(x)
# x = Variable(torch.transpose(x.data, 0, 1))
# print(x) # [torch.FloatTensor of size 64x35x128]
# if self.args.static:
# x = Variable(x.data)
# print(x) # [torch.FloatTensor of size 64x35x128]
# x2 = [F.relu(conv(x)).squeeze(3) for conv in self.convs1] # [(N,Co,W), ...]*len(Ks)
# print(x2)
a = []
for conv in self.convs1:
xx = conv(x) # variable [torch.FloatTensor of size 16x200x35x1]
# print(xx)
xx = Variable(torch.transpose(xx.data, 2, 3))
xx = Variable(torch.transpose(xx.data, 1, 2))
xx = self.bn(xx)
xx = F.relu(xx)
xx = xx.squeeze(1)
a.append(xx)
# print(a)
x = a
# print(x) # [torch.FloatTensor of size 64x100x31],32,33,34
x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # [(N,Co), ...]*len(Ks)
# print(x) # [torch.FloatTensor of size 64x100]*4
x = torch.cat(x, 1)
# print(x) # [torch.FloatTensor of size 64x400]
'''
x1 = self.conv_and_pool(x,self.conv13) #(N,Co)
x2 = self.conv_and_pool(x,self.conv14) #(N,Co)
x3 = self.conv_and_pool(x,self.conv15) #(N,Co)
x = torch.cat((x1, x2, x3), 1) # (N,len(Ks)*Co)
'''
x = self.dropout(x) # (N,len(Ks)*Co)
# print(x) # [torch.FloatTensor of size 64x400]
logit = self.fc1(x) # (N,C)
# print(logit) # [torch.FloatTensor of size 64x2]
return logit
def forward(self, x, y, r, z):
U = self.linear1(z) + self.linear2(x) + self.linear3(y) + self.linear4(r)
result = torch.transpose(self.lin_seq(U), 1, 2)
x = int(math.sqrt(result.size()[2])) # is same as `y`
res = result.view(-1, self.opt.c_dim, x, x)
return res
def forward(self, base_target_emb, input_from_dec, encoder_out_top,
encoder_out_combine):
"""
Args:
base_target_emb: target emb tensor
input_from_dec: output of decode conv
encoder_out_top: the key matrix for calculation of attetion weight,
which is the top output of encode conv
encoder_out_combine:
the value matrix for the attention-weighted sum,
which is the combination of base emb and top output of encode
"""
# checks
# batch, channel, height, width = base_target_emb.size()
batch, _, height, _ = base_target_emb.size()
# batch_, channel_, height_, width_ = input_from_dec.size()
batch_, _, height_, _ = input_from_dec.size()
aeq(batch, batch_)
aeq(height, height_)
# enc_batch, enc_channel, enc_height = encoder_out_top.size()
enc_batch, _, enc_height = encoder_out_top.size()
# enc_batch_, enc_channel_, enc_height_ = encoder_out_combine.size()
enc_batch_, _, enc_height_ = encoder_out_combine.size()
aeq(enc_batch, enc_batch_)
aeq(enc_height, enc_height_)
preatt = seq_linear(self.linear_in, input_from_dec)
target = (base_target_emb + preatt) * SCALE_WEIGHT
target = torch.squeeze(target, 3)
target = torch.transpose(target, 1, 2)
pre_attn = torch.bmm(target, encoder_out_top)
if self.mask is not None:
pre_attn.data.masked_fill_(self.mask, -float('inf'))
attn = F.softmax(pre_attn, dim=2)
context_output = torch.bmm(
attn, torch.transpose(encoder_out_combine, 1, 2))
context_output = torch.transpose(
torch.unsqueeze(context_output, 3), 1, 2)
return context_output, attn
def read(self, k, b):
# k: (1, M_DIM*kr), b: (1, kr)
kr = b.size(1)
K = k.view(kr, M_DIM)
_K = F.normalize(K, eps=EPS)
_M = F.normalize(self.M, eps=EPS)
C = T.matmul(_K, T.transpose(_M, 0, 1))
B = b.repeat(N_mem, 1) # beta
B = T.transpose(B, 0, 1)
if kr == Kr:
self.W_predictor = F.softmax(B*C, dim=1) # B*C: elementwise multiplication
M = T.matmul(self.W_predictor, self.M)
elif kr == Krp:
self.W_policy = F.softmax(B*C, dim=1)
M = T.matmul(self.W_policy, self.M)
else:
raise(ValueError)
return M.view(1, -1)
def forward(self, XD_input, XT_input):
# Tính embedding của XD_input
XD_input = self.embedding_XD(XD_input)
# Tính layer convulution thứ nhất bằng relu
smiles = functional.relu(self.convolution1_XD(torch.transpose(XD_input, 2, 1)))
# Tính layer convulution thứ hai bằng relu
smiles = functional.relu(self.convolution2_XD(smiles))
# Tính layer convulution thứ ba bằng relu
smiles = functional.relu(self.convolution3_XD(smiles))
# Tính layer max pool 1d
smiles = functional.max_pool1d(smiles, kernel_size=smiles.size()[2:])
# Thay đổi shape của kết quả
smiles = smiles.view(smiles.shape[0], smiles.shape[1])
# Tính embedding của XT_input
XT_input = self.embedding_XT(XT_input)
# Tính layer convulution thứ nhất bằng relu
protein = functional.relu(self.convolution1_XT(torch.transpose(XT_input, 2, 1)))
# Tính layer convulution thứ hai bằng relu
protein = functional.relu(self.convolution2_XT(protein))
# Tính layer convulution thứ ba bằng relu
protein = functional.relu(self.convolution3_XT(protein))
# Tính layer max pool 1d
protein = functional.max_pool1d(protein, kernel_size=protein.size()[2:])
# Thay đổi shape của kết quả
protein = protein.view(protein.shape[0], protein.shape[1])
# Gộp hai layer
interaction = torch.cat((smiles, protein), 1)
# Tính layer fully connected 1 bằng relu
f_relu = functional.relu(self.fully_connected1(interaction))
# Tính dropout 1
f_relu = self.dropout1(f_relu)
# Tính layer fully connected 2 bằng relu
f_relu = functional.relu(self.fully_connected2(f_relu))
# Tính dropout 12
f_relu = self.dropout2(f_relu)
# Tính layer fully connected 3 bằng relu
f_relu = functional.relu(self.fully_connected3(f_relu))
# Tính layer fully connected 4
f_relu = self.fully_connected4(f_relu)
# Trả về kết quả
return f_relu
def gen_look_at_matrix(pos, look, up):
d = normalize(look - pos)
right = normalize(torch.cross(d, normalize(up)))
new_up = normalize(torch.cross(right, d))
z = torch.zeros([1], dtype=torch.float32)
o = torch.ones([1], dtype=torch.float32)
return torch.transpose(torch.stack([torch.cat([right , z], 0),
torch.cat([new_up, z], 0),
torch.cat([d , z], 0),
torch.cat([pos , o], 0)]), 0, 1).contiguous()
def forward(self, inputs):
query, title = inputs
queryEmb = self.emb(query)
queryEmb = torch.transpose(queryEmb,0,1)
out, hiden = self.lstm(queryEmb)
queryEnc = out[-1]
titleEmb = self.emb(title)
titleEmb = torch.transpose(titleEmb,0,1)
out, hiden = self.lstm(titleEmb)
titleEnc = out[-1]
mulEnc = queryEnc * titleEnc
addEnc = queryEnc + titleEnc
catEnc = torch.cat([queryEnc, titleEnc, mulEnc, addEnc], 1)
out = self.fc(catEnc)
return out
def backward(self, grad_output):
grad_input1 = self.input1.new(self.input1.size()).zero_()
# if grad_output.is_cuda:
# self.batchgrid = self.batchgrid.cuda()
# grad_input1 = grad_input1.cuda()
grad_input1 = torch.baddbmm(grad_input1, torch.transpose(grad_output.view(-1, self.height*self.width, 2), 1,2), self.batchgrid.view(-1, self.height*self.width, 3))
return grad_input1
def match(self, passage_encoders, question_encoders, wq_matrix, wp_matrix, fw = True):
'''
passage_encoders (pn_steps, batch, hidden_size)
question_encoders (qn_steps, batch, hidden_size)
wq_matrix (qn_steps, batch, hidden_size)
wp_matrix (pn_steps, batch, hidden_size)
'''
if fw:
match_lstm = self.fw_match_lstm
start = 0
end = passage_encoders.size(0)
stride = 1
else:
match_lstm = self.bw_match_lstm
start = passage_encoders.size(0) - 1
end = -1
stride = -1
hx = Variable(torch.zeros(passage_encoders.size(1), self.hidden_size)).cuda()
cx = Variable(torch.zeros(passage_encoders.size(1), self.hidden_size)).cuda()
match_encoders = [0 for i in range(passage_encoders.size(0))]
for i in range(start, end, stride):
wphp = wp_matrix[i]
wrhr = self.whr_net(hx)
_sum = torch.add(wphp, wrhr) # batch, hidden_size
_sum = _sum.expand(wq_matrix.size(0), wq_matrix.size(1), self.hidden_size) # qn_steps, batch, hidden_size
g = self.tanh(torch.add(wq_matrix, _sum)) # qn_steps, batch, hidden_size
g = torch.transpose(g, 0, 1)# batch, qn_steps, hidden_size
wg = self.w_net(g) # bactch, qn_steps, 1
wg = wg.squeeze(-1) # bactch, qn_steps
alpha = wg # bactch, qn_steps
alpha = self.softmax(alpha).view(alpha.size(0), 1, alpha.size(1)) # batch,1, qn_steps
attentionv = torch.bmm(alpha, question_encoders.transpose(0, 1)) # bacth, 1, hidden_size
attentionv = attentionv.squeeze(1) # bacth, hidden_size
inp = torch.cat([passage_encoders[i], attentionv], -1)
hx, cx = match_lstm(inp, (hx, cx)) # batch, hidden_size
match_encoders[i] = hx.view(1, hx.size(0), -1)
match_encoders = torch.cat(match_encoders)
return match_encoders
def ShuffleLayer(x, groups):
batchsize, num_channels, height, width = x.data.size()
channels_per_group = num_channels // groups
### reshape
x = x.view(batchsize, groups,
channels_per_group, height, width)
### transpose
x = torch.transpose(x, 1, 2).contiguous()
### flatten
x = x.view(batchsize, -1, height, width)
return x
def forward(self, x):
embed = self.embed(x)
# CNN
cnn_x = embed
cnn_x = self.dropout(cnn_x)
cnn_x = cnn_x.unsqueeze(1)
cnn_x = [F.relu(conv(cnn_x)).squeeze(3) for conv in self.convs1] # [(N,Co,W), ...]*len(Ks)
cnn_x = torch.cat(cnn_x, 0)
cnn_x = torch.transpose(cnn_x, 1, 2)
# GRU
lstm_out, self.hidden = self.gru(cnn_x, self.hidden)
lstm_out = torch.transpose(lstm_out, 0, 1)
lstm_out = torch.transpose(lstm_out, 1, 2)
lstm_out = F.max_pool1d(lstm_out, lstm_out.size(2)).squeeze(2)
# linear
cnn_lstm_out = self.hidden2label1(F.tanh(lstm_out))
cnn_lstm_out = self.hidden2label2(F.tanh(cnn_lstm_out))
# output
logit = cnn_lstm_out
return logit
def forward(self, sentence: LongTensor, category: LongTensor) -> ACSAModelOutputs:
"""
模型运行
:param sentence: 句子的 index tensor
:param category: category index tensor
:return:
"""
assert sentence.dim() == 2, f"sentence dim: {sentence.dim()} 与 (batch_size, seq_len) 不匹配"
assert category.dim() == 1, f"category dim: {category.dim()} 与 （batch_size,) 不匹配"
bool_mask: BoolTensor = sequence_mask(sequence=sentence,
padding_index=self._token_vocabulary.padding_index)
long_mask = bool_mask.long()
sentence_length = long_mask.sum(dim=-1)
assert sentence_length.dim() == 1, f"sentence_length dim: {sentence_length.dim()} 与 (batch_size,) 不匹配"
# sentence embedding, shape: (batch_size, seq_len, embedding_dim)
sentence_embedding = self.token_embedding(sentence)
assert sentence_embedding.dim() == 3, \
f"sentence_embedding dim: {sentence_embedding.dim()} 与 (batch_size, seq_len, embedding_dim) 不匹配"
# 对 category expand，(batch_size,) -> (batch_size, seq_len)
# category.unsequeeze, (batch_size,) -> (batch_size, 1)
category = category.unsqueeze(dim=1)
# category.expand_as, (batch_size, 1) -> (batch_size, seq_len)
category = category.expand_as(sentence)
# category embedding, shape: (batch_size, seq_len, category_embedding_dim)
category_embedding = self.category_embedding(category)
assert category_embedding.dim() == 3, \
f"category_embedding dim: {category_embedding.dim()} 与 (batch_size, seq_len, category_embedding_dim) 不匹配"
# 将word embedding 与 category embedding 合并在一起
input_embedding = torch.cat((category_embedding, sentence_embedding), dim=-1)
# 使用 lstm sequence encoder 进行 encoder
packed_sentence_embedding = pack_padded_sequence(input=input_embedding,
lengths=sentence_length,
batch_first=True,
enforce_sorted=False)
packed_sequence, (h_n, c_n) = self.lstm(packed_sentence_embedding)
# Tuple, sentence: shape: B * SeqLen * InputSize 和 sentence length
(sentence_encoding, _) = pad_packed_sequence(packed_sequence, batch_first=True)
# h_n shape (num_layers * num_directions, batch_size, hidden_size)
h_n = torch.transpose(h_n, 0, 1)
last_index = -2 if self.lstm.bidirectional else -1
hidden_size = self.lstm.hidden_size * 2 if self.lstm.bidirectional else self.lstm.hidden_size
# hn_last shape: (batch_size, hidden_size * (1 or 2))
hn_last = h_n[:, last_index:, :].contiguous().view(-1, hidden_size)
# 将 lstm 输出与 aspect embedding 合并在一起，准备做 attention
# attention_inputs shape: (batch_size, seq_len, attention_dim = (lstm_hidden_size + category_dim))
attention_inputs = torch.cat((sentence_encoding, category_embedding), dim=-1)
# attention_seq_vec shape: (B, lstm_hidden_size + category_dim)
attention_seq_vec = self.attention_seq2vec(sequence=attention_inputs, mask=long_mask)
# sentiment_vec shape: (B, lstm_hidden_size + category_dim + lstm_hidden_size)
sentiment_vec = torch.cat((attention_seq_vec, hn_last), dim=-1)
logits = self.fc(sentiment_vec)
model_outputs = ACSAModelOutputs(logits=logits)
return model_outputs
def signal_map_params(x,degree):
polynomial = poly(degree, x[:,:,13], x[:,:,12])
beta = torch.mm(torch.inverse(torch.mm(torch.transpose(polynomial,0,1), polynomial)),
torch.mm(torch.transpose(polynomial,0,1), x[:,:,6]))
return beta
def process_image(oriImg, model, model_params, jjac_info):
# for visualize
t0 = time.time()
canvas = np.copy(oriImg)
#imageToTest = Variable(T.transpose(T.transpose(T.unsqueeze(torch.from_numpy(oriImg).float(), 0), 2, 3), 1, 2),
# volatile=True).cuda()
#print oriImg.shape
tic = time.time()
scale = model_params['model_']['boxsize'] / float(oriImg.shape[0])
#print scale
h = int(oriImg.shape[0] * scale)
w = int(oriImg.shape[1] * scale)
pad_h = 0 if (h % model_params['model_']['stride']
== 0) else model_params['model_']['stride'] - (
h % model_params['model_']['stride'])
pad_w = 0 if (w % model_params['model_']['stride']
== 0) else model_params['model_']['stride'] - (
w % model_params['model_']['stride'])
new_h = h + pad_h
new_w = w + pad_w
#print 'scaled width and height ({}, {})'.format(h, w)
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(
imageToTest, model_params['model_']['stride'],
model_params['model_']['padValue'])
imageToTest_padded = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 2, 0, 1)) / 256 - 0.5
feed = Variable(T.from_numpy(imageToTest_padded)).cuda()
output1, output2 = model(feed)
heatmap = nn.UpsamplingBilinear2d(
(oriImg.shape[0], oriImg.shape[1])).cuda()(output2)
paf = nn.UpsamplingBilinear2d(
(oriImg.shape[0], oriImg.shape[1])).cuda()(output1)
toc = time.time()
model_running_time = toc - tic
#print heatmap.size() # (360, 640, 3)
#print paf.size()
#print type(heatmap)
tic = time.time()
heatmap_avg = T.transpose(T.transpose(heatmap[0], 0, 1), 1,
2).data.cpu().numpy()
#paf_avg = T.transpose(T.transpose(paf[0], 0, 1), 1, 2).data.cpu().numpy()
all_peaks = []
peak_counter = 0
# 13-17 are head
# 10, 11, 12, 13 are legs
# hand_parts = [5, 6, 7, 8, 15, 16]
# 4 is right hand
# todo is it actually left elbow below because inverse? confirm
# 5 was left elbow
# 6 was left shoulder
# 7 is left hand. Right hand only?
# 15 is head. right part of head
body_part_index_to_name = {
'4': 'Right Hand',
'5': 'Left Elbow',
'6': 'Left Shoulder',
'7': 'Left Hand',
'15': 'Left Eye',
'14': '1',
'16': '2',
'17': '3'
}
hand_parts = [4, 7, 15, 16, 17]
# hand_parts = list(range(18))
# for part in range(18):
for part in hand_parts:
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down,
map > model_params['param_']['thre1']))
# peaks_binary = T.eq(
# peaks = zip(T.nonzero(peaks_binary)[0],T.nonzero(peaks_binary)[0])
peaks = zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0]) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
# print peaks_with_score_and_id
peak_counter += len(peaks)
#print 'heat map peak stuff time is %.5f' % (time.time() - tic)
# connection_all = []
# special_k = []
# mid_num = 10
# for k in range(len(mapIdx)):
# score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
# candA = all_peaks[limbSeq[k][0] - 1]
# candB = all_peaks[limbSeq[k][1] - 1]
# nA = len(candA)
# nB = len(candB)
# indexA, indexB = limbSeq[k]
# if (nA != 0 and nB != 0):
# connection_candidate = []
# for i in range(nA):
# for j in range(nB):
# vec = np.subtract(candB[j][:2], candA[i][:2])
# norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
# vec = np.divide(vec, norm)
#
# startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
# np.linspace(candA[i][1], candB[j][1], num=mid_num))
#
# vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
# for I in range(len(startend))])
# vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
# for I in range(len(startend))])
#
# score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
# score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(
# 0.5 * oriImg.shape[0] / norm - 1, 0)
# criterion1 = len(np.nonzero(score_midpts > param_['thre2'])[0]) > 0.8 * len(score_midpts)
# criterion2 = score_with_dist_prior > 0
# if criterion1 and criterion2:
# connection_candidate.append(
# [i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]])
#
# connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
# connection = np.zeros((0, 5))
# for c in range(len(connection_candidate)):
# i, j, s = connection_candidate[c][0:3]
# if (i not in connection[:, 3] and j not in connection[:, 4]):
# connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
# if (len(connection) >= min(nA, nB)):
# break
#
# connection_all.append(connection)
# else:
# special_k.append(k)
# connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
# subset = -1 * np.ones((0, 20))
# candidate = np.array([item for sublist in all_peaks for item in sublist])
#
# for k in range(len(mapIdx)):
# if k not in special_k:
# partAs = connection_all[k][:, 0]
# partBs = connection_all[k][:, 1]
# indexA, indexB = np.array(limbSeq[k]) - 1
#
# for i in range(len(connection_all[k])): # = 1:size(temp,1)
# found = 0
# subset_idx = [-1, -1]
# for j in range(len(subset)): # 1:size(subset,1):
# if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
# subset_idx[found] = j
# found += 1
#
# if found == 1:
# j = subset_idx[0]
# if (subset[j][indexB] != partBs[i]):
# subset[j][indexB] = partBs[i]
# subset[j][-1] += 1
# subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
# elif found == 2: # if found 2 and disjoint, merge them
# j1, j2 = subset_idx
# print "found = 2"
# membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
# if len(np.nonzero(membership == 2)[0]) == 0: # merge
# subset[j1][:-2] += (subset[j2][:-2] + 1)
# subset[j1][-2:] += subset[j2][-2:]
# subset[j1][-2] += connection_all[k][i][2]
# subset = np.delete(subset, j2, 0)
# else: # as like found == 1
# subset[j1][indexB] = partBs[i]
# subset[j1][-1] += 1
# subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
#
# # if find no partA in the subset, create a new subset
# elif not found and k < 17:
# row = -1 * np.ones(20)
# row[indexA] = partAs[i]
# row[indexB] = partBs[i]
# row[-1] = 2
# row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
# subset = np.vstack([subset, row])
#
# # delete some rows of subset which has few parts occur
# deleteIdx = [];
# for i in range(len(subset)):
# if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
# deleteIdx.append(i)
# subset = np.delete(subset, deleteIdx, axis=0)
# canvas = cv2.imread(test_image) # B,G,R order
found_hand = False
found_head = False
found_2_hands = False
tic = time.time()
# for i in range(18):
for i in range(len(hand_parts)):
for j in range(len(all_peaks[i])):
#cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
# Right hand is first in loop. so if it fails we have backup
if body_part_index_to_name[str(hand_parts[i])] == 'Right Hand': #
# BGR order. So blue is below
# cv2.circle(canvas, all_peaks[i][j][0:2], 4, (255, 0, 0), thickness=-1)
#print 'Hand position:', all_peaks[i][j][0:2]
found_hand = True
hand_position = all_peaks[i][j][0:2]
elif body_part_index_to_name[str(hand_parts[i])] == 'Left Hand':
# cv2.circle(canvas, all_peaks[i][j][0:2], 4, (0, 255, 0), thickness=-1)
if not found_hand:
print 'Did not find Right hand but found left hand!'
found_hand = True
hand_position = all_peaks[i][j][0:2]
else:
print 'Found two hands'
found_2_hands = True
second_hand_position = all_peaks[i][j][0:2]
#hand_position = all_peaks[i][j][0:2]
elif body_part_index_to_name[str(hand_parts[i])] == 'Left Eye':
# BGR order. So red is below
# cv2.circle(canvas, all_peaks[i][j][0:2], 4, (0, 0, 255), thickness=-1)
#print 'Head position:', all_peaks[i][j][0:2]
found_head = True
head_position = all_peaks[i][j][0:2]
jjac_info['last_x_head_pos'].append(head_position[1])
if len(jjac_info['last_x_head_pos']) > 10:
jjac_info['last_x_head_pos'].pop()
biggest_diff = max(jjac_info['last_x_head_pos']) - min(
jjac_info['last_x_head_pos'])
if biggest_diff > jjac_info['biggest_diff']:
jjac_info['biggest_diff'] = biggest_diff
# espeak_command = 'Largest y range from last_x_head_pos: {}'.format(biggest_diff)
espeak_command = 'Largest range from last positions: {}'.format(
biggest_diff)
print espeak_command
# os.system(espeak_command)
# jjac_info['head_y_range']
elif body_part_index_to_name[str(hand_parts[i])] == '1':
cv2.circle(canvas,
all_peaks[i][j][0:2],
4, (0, 0, 255),
thickness=-1)
elif body_part_index_to_name[str(hand_parts[i])] == '2':
cv2.circle(canvas,
all_peaks[i][j][0:2],
4, (0, 255, 255),
thickness=-1)
elif body_part_index_to_name[str(hand_parts[i])] == '3':
cv2.circle(canvas,
all_peaks[i][j][0:2],
4, (255, 255, 255),
thickness=-1)
else:
# cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
cv2.circle(canvas,
all_peaks[i][j][0:2],
4, (128, 128, 128),
thickness=-1)
#global how_many_times_hands_went_over_head, hands_over_head
if found_hand and found_head:
# both hands over head or 1 hand over head
if (found_2_hands and hand_position[1] < head_position[1]
and second_hand_position[1] < head_position[1]
) or hand_position[1] < head_position[1]:
print 'Hand is higher than head {} < {}'.format(
hand_position[1], head_position[1])
# if first time hand over head, then increase jumping jack count
if not jjac_info['hands_over_head']:
jjac_info['hands_over_head'] = True
jjac_info['num_jumping_jacks'] += 1
print 'Number of jumping jacks:', jjac_info[
'num_jumping_jacks']
# speak speech command example
# espeak '1 Jumping Jack'
espeak_command = "espeak {}".format(
'\'{} Jumping jack{}\''.format(
jjac_info['num_jumping_jacks'],
's' if jjac_info['num_jumping_jacks'] > 1 else ''))
# os.system(espeak_command) # slows it down?
else:
jjac_info['hands_over_head'] = False
print 'hand is lower than head {} > {}'.format(
hand_position[1], head_position[1])
#print 'Drawing circles time is %.5f' % (time.time() - tic)
cv2.putText(
canvas,
"No. Jumping Jacks: {} ".format(jjac_info['num_jumping_jacks']),
(0, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.6, 255)
cv2.putText(
canvas, "Hands over head: {}".format(
'Yes' if jjac_info['hands_over_head'] else 'No'), (0, 45),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, 255)
# stickwidth = 4
# for i in range(17):
# for n in range(len(subset)):
# index = subset[n][np.array(limbSeq[i]) - 1]
# if -1 in index:
# continue
# cur_canvas = canvas.copy()
# Y = candidate[index.astype(int), 0]
# X = candidate[index.astype(int), 1]
# mX = np.mean(X)
# mY = np.mean(Y)
# length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
# angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
# polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
# cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
# canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
print 'cv and print: %.5f, Model time: %.5f, Full processing time is %.5f.' % (
time.time() - tic, model_running_time, time.time() - t0)
return canvas
def handle_one(oriImg):
# for visualize
canvas = np.copy(oriImg)
#canvas = np.ones(canvas.shape)*255
imageToTest = Variable(T.transpose(T.transpose(T.unsqueeze(torch.from_numpy(oriImg).float(),0),2,3),1,2),volatile=True).cuda()
#print(oriImg.shape)
scale = model_['boxsize'] / float(oriImg.shape[0])
#print (scale)
h = int(oriImg.shape[0]*scale)
w = int(oriImg.shape[1]*scale)
pad_h = 0 if (h%model_['stride']==0) else model_['stride'] - (h % model_['stride'])
pad_w = 0 if (w%model_['stride']==0) else model_['stride'] - (w % model_['stride'])
new_h = h+pad_h
new_w = w+pad_w
imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = utill.padRightDownCorner(imageToTest, model_['stride'], model_['padValue'])
imageToTest_padded = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,2,0,1))/256 - 0.5
feed = Variable(T.from_numpy(imageToTest_padded)).cuda()
output1,output2 = model(feed)
heatmap = nn.UpsamplingBilinear2d((oriImg.shape[0], oriImg.shape[1])).cuda()(output2)
paf = nn.UpsamplingBilinear2d((oriImg.shape[0], oriImg.shape[1])).cuda()(output1)
#print (heatmap.size())
#print (paf.size())
#print (type(heatmap))
heatmap_avg = T.transpose(T.transpose(heatmap[0],0,1),1,2).data.cpu().numpy()
paf_avg = T.transpose(T.transpose(paf[0],0,1),1,2).data.cpu().numpy()
all_peaks = []
peak_counter = 0
#maps =
for part in range(18):
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:]
peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param_['thre1']))
# peaks_binary = T.eq(
# peaks = zip(T.nonzero(peaks_binary)[0],T.nonzero(peaks_binary)[0])
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0]-1]
candB = all_peaks[limbSeq[k][1]-1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if(nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1])
if norm==0:
norm=0.000000000000000000001
vec = np.divide(vec, norm)
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*oriImg.shape[0]/norm-1, 0)
criterion1 = len(np.nonzero(score_midpts > param_['thre2'])[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]])
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0,5))
for c in range(len(connection_candidate)):
i,j,s = connection_candidate[c][0:3]
if(i not in connection[:,3] and j not in connection[:,4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
if(len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:,0]
partBs = connection_all[k][:,1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if(subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
#print ("found = 2")
membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
canvas = np.ones(canvas.shape)*255
# canvas = cv2.imread(test_image) # B,G,R order
for i in range(18):
for j in range(len(all_peaks[i])):
cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
stickwidth = 4
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i])-1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
return canvas
def forward(self,
sentence,
p_sentence,
pos_tags,
lengths,
target_idx_in,
region_marks,
local_roles_voc,
frames,
local_roles_mask,
sent_pred_lemmas_idx,
dep_tags,
dep_heads,
targets,
test=False):
embeds = self.word_embeddings(sentence)
embeds = embeds.view(self.batch_size, len(sentence[0]),
self.word_emb_dim)
pos_embeds = self.pos_embeddings(pos_tags)
fixed_embeds = self.word_fixed_embeddings(p_sentence)
fixed_embeds = fixed_embeds.view(self.batch_size, len(sentence[0]),
self.word_emb_dim)
sent_pred_lemmas_embeds = self.p_lemma_embeddings(sent_pred_lemmas_idx)
region_marks = region_marks.view(self.batch_size, len(sentence[0]), 1)
embeds = torch.cat((embeds, fixed_embeds, pos_embeds,
sent_pred_lemmas_embeds, region_marks), 2)
#embeds = torch.cat((embeds, fixed_embeds, pos_embeds, region_marks), 2)
# share_layer
embeds_sort, lengths_sort, unsort_idx = self.sort_batch(
embeds, lengths)
embeds_sort = rnn.pack_padded_sequence(embeds_sort,
lengths_sort,
batch_first=True)
# hidden states [time_steps * batch_size * hidden_units]
hidden_states, self.hidden = self.BiLSTM_share(embeds_sort,
self.hidden)
# it seems that hidden states is already batch first, we don't need swap the dims
# hidden_states = hidden_states.permute(1, 2, 0).contiguous().view(self.batch_size, -1, )
hidden_states, lens = rnn.pad_packed_sequence(hidden_states,
batch_first=True)
#hidden_states = hidden_states.transpose(0, 1)
hidden_states = hidden_states[unsort_idx]
forward_h, backward_h = torch.split(hidden_states, self.hidden_dim, 2)
forward_e = forward_h[:, :, :50]
backward_e = backward_h[:, :, :50]
bf_e = torch.cat((forward_e, backward_e), 2)
concat_embeds = torch.zeros(bf_e.size()[0],
bf_e.size()[1],
bf_e.size()[2])
for i in range(bf_e.size()[0]):
for j in range(bf_e.size()[1]):
if dep_heads[i][j] > 0:
concat_embeds[i, j] = bf_e[i, dep_heads[i][j] - 1]
bf_e = torch.cat((bf_e, concat_embeds), 2)
dep_tag_space = self.MLP(F.tanh(self.hidden2tag(bf_e))).view(
len(sentence[0]) * self.batch_size, -1)
# SRL layer
embeds_sort, lengths_sort, unsort_idx = self.sort_batch(
hidden_states, lengths)
embeds_sort = rnn.pack_padded_sequence(embeds_sort,
lengths_sort.cpu().numpy(),
batch_first=True)
# hidden states [time_steps * batch_size * hidden_units]
hidden_states, self.hidden_2 = self.BiLSTM_SRL(embeds_sort,
self.hidden_2)
# it seems that hidden states is already batch first, we don't need swap the dims
# hidden_states = hidden_states.permute(1, 2, 0).contiguous().view(self.batch_size, -1, )
hidden_states, lens = rnn.pad_packed_sequence(hidden_states,
batch_first=True)
# hidden_states = hidden_states.transpose(0, 1)
hidden_states = hidden_states[unsort_idx]
# B * H
hidden_states_3 = hidden_states
predicate_embeds = hidden_states_3[
np.arange(0,
hidden_states_3.size()[0]), target_idx_in]
# T * B * H
added_embeds = Variable(
torch.zeros(hidden_states_3.size()[1],
hidden_states_3.size()[0],
hidden_states_3.size()[2]))
predicate_embeds = added_embeds + predicate_embeds
# B * T * H
predicate_embeds = predicate_embeds.transpose(0, 1)
hidden_states = torch.cat((hidden_states_3, predicate_embeds), 2)
# print(hidden_states)
# non-linear map and rectify the roles' embeddings
# roles = Variable(torch.from_numpy(np.arange(0, self.tagset_size)))
# B * roles
# log(local_roles_voc)
# log(frames)
# B * roles * h
role_embeds = self.role_embeddings(local_roles_voc)
frame_embeds = self.frame_embeddings(frames)
role_embeds = torch.cat((role_embeds, frame_embeds), 2)
mapped_roles = F.relu(self.role_map(role_embeds))
mapped_roles = torch.transpose(mapped_roles, 1, 2)
# b, times, roles
tag_space = torch.matmul(hidden_states, mapped_roles)
#tag_space = hidden_states.mm(mapped_roles)
# b, roles
#sub = torch.div(torch.add(local_roles_mask, -1.0), _BIG_NUMBER)
sub = torch.add(local_roles_mask, -1.0) * _BIG_NUMBER
sub = torch.FloatTensor(sub.numpy())
# b, roles, times
tag_space = torch.transpose(tag_space, 0, 1)
tag_space += sub
# b, T, roles
tag_space = torch.transpose(tag_space, 0, 1)
tag_space = tag_space.view(len(sentence[0]) * self.batch_size, -1)
SRLprobs = F.softmax(tag_space, dim=1)
wrong_l_nums = 0.0
all_l_nums = 0.0
dep_labels = np.argmax(dep_tag_space.data.numpy(), axis=1)
for predict_l, gold_l in zip(dep_labels,
dep_tags.view(-1).data.numpy()):
if gold_l != 0:
all_l_nums += 1
if predict_l != gold_l and gold_l != 0:
wrong_l_nums += 1
#loss_function = nn.NLLLoss(ignore_index=0)
targets = targets.view(-1)
#tag_scores = F.log_softmax(tag_space)
#loss = loss_function(tag_scores, targets)
loss_function = nn.CrossEntropyLoss(ignore_index=0)
SRLloss = loss_function(tag_space, targets)
DEPloss = loss_function(dep_tag_space, dep_tags.view(-1))
loss = SRLloss + 0.1 * DEPloss
return SRLloss, DEPloss, loss, SRLprobs, wrong_l_nums, all_l_nums
batch_size=batchSize,
shuffle=False)
# print(net)
loadPath = './Project/param'
appendix = '.t7'
# print(IO.exists(loadPath))
Ensemble_num = 10 # Smaller than batchSize ( number of samples used in each batch )
n = 1 # w index
net = Net()
path = loadPath + str(n) + appendix
model_dict=net.load_state_dict(torch.load(path))
for images, labels in test_loader:
outputs = net(images)
print(len(FC1))
# Dimensionality of FC layers
h_fc1_array = torch.zeros(len(FC1)*Ensemble_num,64)
for i in range(len(FC1)):
for j in range(Ensemble_num): # channel idx = 1
h_fc1_array[i*Ensemble_num+j,:] = FC1[i][j].reshape(1,64)
h_fc1_array_T = torch.transpose(h_fc1_array,0,1)
CMatrix = torch.mm(h_fc1_array_T,h_fc1_array)/h_fc1_array.size(0)
h_fc1_array_mean = torch.mean(h_fc1_array,0).view(1,h_fc1_array.size(1))
h_fc1_array_mean_T = torch.transpose(h_fc1_array_mean,0,1)
CMatrix_ = torch.mm(h_fc1_array_mean_T,h_fc1_array_mean)
C = CMatrix - CMatrix_
D = np.square(torch.trace(C).detach().numpy())/torch.trace(torch.mm(C,C)).detach().numpy()
print('Dimensionality of fc layer1 is {0}'.format(D))
else:
for s in state:
s.detach_()
print(X, X.size()) # 32*35
inputs = to_onehot(X, vocab_size)
print(inputs, len(inputs), inputs[0].size()
) # 35, 32*1027, 时间步为35，batch为32，词汇量为1027，35个32*1027的列表
# outputs有num_steps个形状为(batch_size, vocab_size)的矩阵
(outputs, state) = rnn(inputs, state, params)
# 拼接之后形状为(num_steps * batch_size, vocab_size)
outputs = torch.cat(outputs, dim=0)
# Y的形状是(batch_size, num_steps)，转置后再变成长度为
# batch * num_steps 的向量，这样跟输出的行一一对应
print(Y, Y.size()) # 32*35
print('-' * 100)
print(torch.transpose(Y, 0, 1), torch.transpose(Y, 0, 1).size()) # 35*32
print('-' * 100)
print(
torch.transpose(Y, 0, 1).contiguous(),
torch.transpose(Y, 0, 1).contiguous().size()) # 35*32
y = torch.transpose(Y, 0, 1).contiguous().view(-1)
print('-' * 100)
print(y, y.size()) # 1120
print(y.long())
print(outputs, outputs.size()) # 1120*1027
print('=' * 100)
# import math
# 验证交叉熵函数
# entroy=nn.CrossEntropyLoss()
# input=torch.Tensor([[-0.7715, -0.6205, -0.2562],
def batch_distance_matrix_general(A, B):
r_A = torch.sum(A * A, dim=-1).unsqueeze_(dim=-1)
r_B = torch.sum(B * B, dim=-1).unsqueeze_(dim=-1)
m = torch.matmul(A, torch.transpose(B, -1, -2))
D = r_A - 2 * m + torch.transpose(r_B, -1, -2)
return D
def encode_whole_protein(seq,
true_coords,
angles,
padding_seq,
needed_info={
"cutoffs": [2, 5, 10],
"bond_scales": [0.5, 1, 2]
},
free_mem=False):
""" Encodes a whole protein. In points + vectors. """
device, precise = true_coords.device, true_coords.type()
#################
# encode points #
#################
cloud_mask = torch.tensor(scn_cloud_mask(
seq[:-padding_seq or None])).bool().to(device)
flat_mask = rearrange(cloud_mask, 'l c -> (l c)')
# embedd everything
# general position embedding
center_coords = true_coords - true_coords.mean(dim=0)
pos_unit_norms = torch.norm(center_coords, dim=-1, keepdim=True)
pos_unit_vecs = center_coords / pos_unit_norms
pos_unit_norms_enc = encode_dist(
pos_unit_norms, scales=needed_info["atom_pos_scales"]).squeeze()
# reformat coordinates to scn (L, 14, 3) - TODO: solve if padding=0
coords_wrap = rearrange(center_coords, '(l c) d -> l c d',
c=14)[:-padding_seq or None]
# position in backbone embedding
aa_pos = encode_dist(torch.arange(len(seq[:-padding_seq or None]),
device=device).float(),
scales=needed_info["aa_pos_scales"])
atom_pos = chain2atoms(aa_pos)[cloud_mask]
# atom identity embedding
atom_id_embedds = torch.stack([
SUPREME_INFO[k]["atom_id_embedd"] for k in seq[:-padding_seq or None]
],
dim=0)[cloud_mask].to(device)
# aa embedding
seq_int = torch.tensor([AAS2NUM[aa] for aa in seq[:-padding_seq or None]],
device=device).long()
aa_id_embedds = chain2atoms(seq_int, mask=cloud_mask)
# CA - SC distance
dist2ca_vec, dist2ca_norm = dist2ca(coords_wrap)
dist2ca_norm_enc = encode_dist(
dist2ca_norm, scales=needed_info["dist2ca_norm_scales"]).squeeze()
# BACKBONE feats
vecs, norms = orient_aa(coords_wrap)
bb_vecs_atoms = chain2atoms(torch.transpose(vecs, 0, 1), mask=cloud_mask)
bb_norms_atoms = chain2atoms(torch.transpose(norms, 0, 1), mask=cloud_mask)
bb_norms_atoms_enc = encode_dist(bb_norms_atoms, scales=[0.5])
################
# encode bonds #
################
bond_info = encode_whole_bonds(x=coords_wrap[cloud_mask],
x_format="coords",
embedd_info={},
needed_info=needed_info)
whole_bond_idxs, whole_bond_enc, bond_embedd_info = bond_info
#########
# merge #
#########
# concat so that final is [vector_dims, scalar_dims]
point_n_vectors = 1 + 1 + 5
point_n_scalars = 2*len(needed_info["atom_pos_scales"]) + 1 +\
2*len(needed_info["aa_pos_scales"]) + 1 +\
2*len(needed_info["dist2ca_norm_scales"]) + 1+\
rearrange(bb_norms_atoms_enc, 'atoms feats encs -> atoms (feats encs)').shape[1] +\
2 # the last 2 are to be embedded yet
whole_point_enc = torch.cat(
[
pos_unit_vecs[:-padding_seq * 14 or None][flat_mask], # 1
dist2ca_vec[cloud_mask], # 1
rearrange(bb_vecs_atoms, 'atoms n d -> atoms (n d)'), # 5
# scalars
pos_unit_norms_enc[:-padding_seq * 14 or None][flat_mask], # 2n+1
atom_pos, # 2n+1
dist2ca_norm_enc[cloud_mask], # 2n+1
rearrange(bb_norms_atoms_enc,
'atoms feats encs -> atoms (feats encs)'), # 2n+1
atom_id_embedds.unsqueeze(-1),
aa_id_embedds.unsqueeze(-1)
],
dim=-1) # the last 2 are yet to be embedded
if free_mem:
del pos_unit_vecs, dist2ca_vec, bb_vecs_atoms, pos_unit_norms_enc, cloud_mask,\
atom_pos, dist2ca_norm_enc, bb_norms_atoms_enc, atom_id_embedds, aa_id_embedds
# record embedding dimensions
point_embedd_info = {
"point_n_vectors": point_n_vectors,
"point_n_scalars": point_n_scalars,
}
embedd_info = {**point_embedd_info, **bond_embedd_info}
return whole_point_enc, whole_bond_idxs, whole_bond_enc, embedd_info
def inv_transform(self, transformed_inputs):
inputs = torch.transpose(transformed_inputs, 2, 3)
return inputs
def transform(self, inputs):
transformed_inputs = torch.transpose(inputs, 2, 3)
return transformed_inputs
def __call__(self, x):
assert isinstance(x, torch.Tensor)
if x.shape[-2] > x.shape[-1]:
x = torch.transpose(x, -2, -1)
return x
while True:
print("*** GENERATION PHASE ***")
_input = input("Begin with:\t")
if len(_input) <= 0:
_input = []
_input = [i for i in _input]
ids = []
for w in _input:
ids.append(vocab.get_idx(w))
for i in ids:
print(vocab.idx2word[i], end="")
print("")
poetry = torch.tensor([[vocab.word2idx["<eos>"]] + ids
]).type(torch.int64).to(device)
poetry = torch.transpose(poetry, 1, 0)
while len(ids) < max_len:
with torch.no_grad():
output = forward_model(poetry)
output = output.view(-1, ntokens)
nxt_w = random.sample(
list(torch.argsort(output[-1])[-n_rand:].cpu().numpy()), 1)
#print (list(torch.argsort(output[-1])[-n_rand:].cpu().numpy()), nxt_w)
nxt_w = nxt_w[0]
if nxt_w == vocab.word2idx["<eos>"]:
break
ids.append(nxt_w)
for i in ids:
print(vocab.idx2word[i], end="")
print("")
def forward(self, x):
x = self.model.extract_features(x)
x = self.avgpool(x)
x = self.dropout(x)
x = torch.transpose(x, 1, 2).squeeze()
partA, partB, partC, partD = {}, {}, {}, {}
predictA, predictB, predictC, predictD = {}, {}, {}, {}
y = {}
y['PCB'] = []
if self.opt.single_cls:
feat = self.bottleneck(x)
y = self.classifier(feat)
if self.opt.use_triplet_loss:
return y, feat
else:
return y
else:
for i in range(self.opt.nparts):
partA[i] = torch.flatten(x[:, i:i + 1, :], 1)
name = 'classifierA' + str(i)
c = getattr(self, name)
predictA[i] = c(partA[i])
y['PCB'].append(predictA[i])
for i in range(self.opt.nparts - 1):
partB[i] = torch.flatten(x[:, i:i + 2, :], 1)
name = 'classifierB' + str(i)
c = getattr(self, name)
predictB[i] = c(partB[i])
y['PCB'].append(predictB[i])
for i in range(self.opt.nparts - 2):
partC[i] = torch.flatten(x[:, i:i + 3, :], 1)
name = 'classifierC' + str(i)
c = getattr(self, name)
predictC[i] = c(partC[i])
y['PCB'].append(predictC[i])
for i in range(self.opt.nparts - 3):
partD[i] = torch.flatten(x[:, i:i + 4, :], 1)
name = 'classifierD' + str(i)
c = getattr(self, name)
predictD[i] = c(partD[i])
y['PCB'].append(predictD[i])
partB[3] = torch.flatten(torch.cat((x[:, :1, :], x[:, 2:3, :]), 1),
1)
predictB[3] = self.classifierB3(partB[3])
y['PCB'].append(predictB[3])
partB[4] = torch.flatten(torch.cat((x[:, :1, :], x[:, 3:4, :]), 1),
1)
predictB[4] = self.classifierB4(partB[4])
y['PCB'].append(predictB[4])
partB[5] = torch.flatten(
torch.cat((x[:, 1:2, :], x[:, 3:4, :]), 1), 1)
predictB[5] = self.classifierB5(partB[5])
y['PCB'].append(predictB[5])
partC[2] = torch.flatten(torch.cat((x[:, :2, :], x[:, 3:4, :]), 1),
1)
predictC[2] = self.classifierC2(partC[2])
y['PCB'].append(predictC[2])
partC[3] = torch.flatten(torch.cat((x[:, :1, :], x[:, 2:, :]), 1),
1)
predictC[3] = self.classifierC3(partC[3])
y['PCB'].append(predictC[3])
if self.opt.use_triplet_loss:
return y, feat
else:
return y
def main():
# Step 1: init data folders
print("init data folders")
# init character folders for dataset construction
metatrain_character_folders,metatest_character_folders = tg.omniglot_character_folders()
# Step 2: init neural networks
print("init neural networks")
feature_encoder = CNNEncoder()
relation_network = RelationNetwork(FEATURE_DIM,RELATION_DIM)
feature_encoder.cuda(GPU)
relation_network.cuda(GPU)
feature_encoder_optim = torch.optim.Adam(feature_encoder.parameters(),lr=LEARNING_RATE)
feature_encoder_scheduler = StepLR(feature_encoder_optim,step_size=100000,gamma=0.5)
relation_network_optim = torch.optim.Adam(relation_network.parameters(),lr=LEARNING_RATE)
relation_network_scheduler = StepLR(relation_network_optim,step_size=100000,gamma=0.5)
if os.path.exists(str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot.pkl")):
feature_encoder.load_state_dict(torch.load(str("./models/omniglot_feature_encoder_" + str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot.pkl")))
print("load feature encoder success")
if os.path.exists(str("./models/omniglot_relation_network_"+ str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot.pkl")):
relation_network.load_state_dict(torch.load(str("./models/omniglot_relation_network_"+ str(CLASS_NUM) +"way_" + str(SAMPLE_NUM_PER_CLASS) +"shot.pkl")))
print("load relation network success")
total_accuracy = 0.0
for episode in range(EPISODE):
# test
print("Testing...")
total_rewards = 0
accuracies = []
for i in range(TEST_EPISODE):
degrees = random.choice([0,90,180,270])
task = tg.OmniglotTask(metatest_character_folders,CLASS_NUM,SAMPLE_NUM_PER_CLASS,SAMPLE_NUM_PER_CLASS,)
sample_dataloader = tg.get_data_loader(task,num_per_class=SAMPLE_NUM_PER_CLASS,split="train",shuffle=False,rotation=degrees)
test_dataloader = tg.get_data_loader(task,num_per_class=SAMPLE_NUM_PER_CLASS,split="test",shuffle=True,rotation=degrees)
sample_images,sample_labels = sample_dataloader.__iter__().next()
test_images,test_labels = test_dataloader.__iter__().next()
# calculate features
sample_features = feature_encoder(Variable(sample_images).cuda(GPU)) # 5x64
test_features = feature_encoder(Variable(test_images).cuda(GPU)) # 20x64
# calculate relations
# each batch sample link to every samples to calculate relations
# to form a 100x128 matrix for relation network
sample_features_ext = sample_features.unsqueeze(0).repeat(SAMPLE_NUM_PER_CLASS*CLASS_NUM,1,1,1,1)
test_features_ext = test_features.unsqueeze(0).repeat(SAMPLE_NUM_PER_CLASS*CLASS_NUM,1,1,1,1)
test_features_ext = torch.transpose(test_features_ext,0,1)
relation_pairs = torch.cat((sample_features_ext,test_features_ext),2).view(-1,FEATURE_DIM*2,5,5)
relations = relation_network(relation_pairs).view(-1,CLASS_NUM)
_,predict_labels = torch.max(relations.data,1)
rewards = [1 if predict_labels[j].cuda()==test_labels[j].cuda() else 0 for j in range(CLASS_NUM)]
total_rewards += np.sum(rewards)
accuracy = np.sum(rewards)/1.0/CLASS_NUM/SAMPLE_NUM_PER_CLASS
accuracies.append(accuracy)
test_accuracy,h = mean_confidence_interval(accuracies)
print("test accuracy:",test_accuracy,"h:",h)
total_accuracy += test_accuracy
print("aver_accuracy:",total_accuracy/EPISODE)
support = preprocess_adj(adj)
num_supports = 1
model_func = GCN
values = features.data
indices = np.vstack((features.row, features.col))
i = torch.LongTensor(indices)
v = torch.FloatTensor(values)
shape = features.shape
t_features = torch.sparse.FloatTensor(i, v, torch.Size(shape))
t_features = t_features.float()
t_y_train = torch.from_numpy(y_train)
t_y_val = torch.from_numpy(y_val)
t_y_test = torch.from_numpy(y_test)
t_train_mask = torch.from_numpy(train_mask.astype(np.float32))
tm_train_mask = torch.transpose(torch.unsqueeze(t_train_mask, 0), 1,
0).repeat(1, y_train.shape[1])
values = support.data
indices = np.vstack((support.row, support.col))
i = torch.LongTensor(indices)
v = torch.FloatTensor(values)
shape = support.shape
t_support = torch.sparse.FloatTensor(i, v, torch.Size(shape))
# if torch.cuda.is_available():
# model_func = model_func.cuda()
# t_features = t_features.cuda()
# t_y_train = t_y_train.cuda()
# t_y_val = t_y_val.cuda()
# t_y_test = t_y_test.cuda()
# t_train_mask = t_train_mask.cuda()
def BHWC_to_BCHW(tensor):
tensor = torch.transpose(tensor, 1, 3) # BCWH
tensor = torch.transpose(tensor, 2, 3) # BCHW
return tensor
def main():
utils.print_stage("Program Starts")
metatrain_folders, metatest_folders = tg.HAA_folders()
# i3d == the I3D network
encoder = cnn()
# rn == the relation network
rn = RN(FEATURE_DIM, RELATION_DIM)
# Move the model to GPU
encoder.to(device)
rn.to(device)
# Define Optimizer
encoder_optim = torch.optim.Adam(encoder.parameters(), lr=LEARNING_RATE)
rn_optim = torch.optim.Adam(rn.parameters(), lr=LEARNING_RATE)
# Define Scheduler
encoder_scheduler = StepLR(encoder_optim, step_size=100000, gamma=0.5)
rn_scheduler = StepLR(rn_optim, step_size=100000, gamma=0.5)
# Load Saved Model #TODO
# Training Starts Here
utils.print_stage("Start Training")
# Accuracy Record
max_accuracy = 0 #TODO read accuracy from somewhere
for episode in range(EPISODE):
print("{}\tepisode{}".format(max_accuracy, episode))
# Update "step" for scheduler
rn_scheduler.step(episode)
encoder_scheduler.step(episode)
# Setup Data #TODO
task = tg.HAATask(metatrain_folders, CLASS_NUM, SAMPLE_NUM_PER_CLASS,
BATCH_NUM_PER_CLASS)
sample_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="train",
shuffle=False)
batch_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=BATCH_NUM_PER_CLASS,
split="test",
shuffle=True)
samples, sample_labels = sample_dataloader.__iter__().next(
) #25*3*84*84
batches, batch_labels = batch_dataloader.__iter__().next()
# Encoding
sample_features = encoder(
Variable(samples).to(device)) # 25*64*19*19 #
sample_features = sample_features.view(CLASS_NUM, SAMPLE_NUM_PER_CLASS,
FEATURE_DIM, 62, 62) # TODO
sample_features = torch.sum(sample_features, 1).squeeze(1) #
batch_features = encoder(Variable(batches).to(
device)) # 20x64*5*5 #
# Compute Relation
sample_features_ext = sample_features.unsqueeze(0).repeat(
BATCH_NUM_PER_CLASS * CLASS_NUM, 1, 1, 1, 1) #
batch_features_ext = batch_features.unsqueeze(0).repeat(
CLASS_NUM, 1, 1, 1, 1) #
batch_features_ext = torch.transpose(batch_features_ext, 0, 1) # TODO
relation_pairs = torch.cat((sample_features_ext, batch_features_ext),
2).view(-1, FEATURE_DIM * 2, 62, 62) #
relations = rn(relation_pairs).view(-1, CLASS_NUM) #
# Compute Loss
mse = nn.MSELoss().to(device) #
one_hot_labels = Variable(
torch.zeros(BATCH_NUM_PER_CLASS * CLASS_NUM,
CLASS_NUM).scatter_(1, batch_labels.view(-1, 1),
1).to(device)) # TODO
loss = mse(relations, one_hot_labels) #
# Train Model
encoder.zero_grad()
rn.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(encoder.parameters(), 0.5)
nn.utils.clip_grad_norm(rn.parameters(), 0.5)
encoder_optim.step()
rn_optim.step()
# Periodically Print Loss #TODO
# Testing
if (episode % 200 == 0 and episode != 0) or episode == EPISODE - 1:
utils.print_stage("Testing")
accuracies = []
for _ in range(TEST_EPISODE):
# Data Loading #TODO
total_rewards = 0
task = tg.HAATask(metatest_folders, CLASS_NUM,
SAMPLE_NUM_PER_CLASS, 15) #
sample_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=SAMPLE_NUM_PER_CLASS,
split="train",
shuffle=False) #
num_per_class = 5 # TODO
test_dataloader = tg.get_HAA_data_loader(
task,
num_per_class=num_per_class,
split="test",
shuffle=False) #
sample_images, sample_labels = sample_dataloader.__iter__(
).next() #
for test_images, test_labels in test_dataloader:
batch_size = test_labels.shape[0]
# Encoding
sample_features = encoder(
Variable(sample_images).to(
device)) # 5x64 #
sample_features = sample_features.view(
CLASS_NUM, SAMPLE_NUM_PER_CLASS, FEATURE_DIM, 62,
62) # TODO
sample_features = torch.sum(sample_features,
1).squeeze(1) #
test_features = encoder(Variable(test_images).to(
device)) # 20x64 #
# Compute Relation
sample_features_ext = sample_features.unsqueeze(0).repeat(
batch_size, 1, 1, 1, 1) #
test_features_ext = test_features.unsqueeze(0).repeat(
1 * CLASS_NUM, 1, 1, 1, 1) #
test_features_ext = torch.transpose(
test_features_ext, 0, 1) # TODO
relation_pairs = torch.cat(
(sample_features_ext, test_features_ext),
2).view(-1, FEATURE_DIM * 2, 62, 62) #
relations = rn(relation_pairs).view(-1, CLASS_NUM) #
# Predict
_, predict_labels = torch.max(relations.data, 1)
# Counting Correct Ones
rewards = [
1 if predict_labels[j] == test_labels[j] else 0
for j in range(batch_size)
]
total_rewards += np.sum(rewards)
# Record accuracy
accuracy = total_rewards / 1.0 / CLASS_NUM / 15
accuracies.append(accuracy)
# Overall accuracy
test_accuracy, _ = utils.mean_confidence_interval(accuracies)
# Save Model
if test_accuracy > max_accuracy:
# save networks
torch.save(
encoder.state_dict(),
str("./models/encoder_" + str(CLASS_NUM) + "way_" +
str(SAMPLE_NUM_PER_CLASS) + "shot_" +
str(test_accuracy) + ".pkl"))
torch.save(
rn.state_dict(),
str("./models/rn_" + str(CLASS_NUM) + "way_" +
str(SAMPLE_NUM_PER_CLASS) + "shot_" +
str(test_accuracy) + ".pkl"))
max_accuracy = test_accuracy
print("Model Saved with accuracy={}".format(max_accuracy))
print("final accuracy = {}".format(max_accuracy))
def setting_samples_from_windows(self, targets, predictions_test,
targets_files):
#logging.info(" Network_User: Segmentation: "
# "with these number of files {}".format(int(torch.max(targets_files).item())))
for target_files in range(0, 1 + int(torch.max(targets_files).item())):
targets_per_file = targets[targets_files == target_files]
predictions_per_file = predictions_test[targets_files == target_files]
size_samples = self.config["sliding_window_step"] * (targets_per_file.size(0) - 1) + \
self.config["sliding_window_length"]
sample_targets = torch.zeros(
(self.config["num_classes"], size_samples)).to(self.device,
dtype=torch.long)
sample_predictions = torch.zeros(
(self.config["num_classes"], size_samples)).to(self.device,
dtype=torch.long)
for ws in range(targets_per_file.size(0)):
window_samples = targets_per_file[ws]
window_samples = nn.functional.one_hot(
window_samples.type(torch.LongTensor),
num_classes=self.config["num_classes"])
window_samples = torch.transpose(window_samples, 0,
1).to(self.device,
dtype=torch.long)
sample_targets[:, self.config["sliding_window_step"] * ws:(
self.config["sliding_window_step"] *
ws) + self.config["sliding_window_length"]] += window_samples
window_samples = torch.ones(
(targets_per_file[ws].size())).to(self.device,
dtype=torch.long)
window_samples = window_samples * predictions_per_file[ws]
window_samples = nn.functional.one_hot(
window_samples.type(torch.LongTensor),
num_classes=self.config["num_classes"])
window_samples = torch.transpose(window_samples, 0,
1).to(self.device,
dtype=torch.long)
sample_predictions[:, self.config["sliding_window_step"] * ws:(
self.config["sliding_window_step"] *
ws) + self.config["sliding_window_length"]] += window_samples
#sample_targets_single = torch.zeros(size_samples)
#sample_predictions_single = torch.zeros(size_samples)
#for ws in range(size_samples):
#bincounts = torch.bincount(sample_targets[:, ws].type(dtype=torch.long),
# minlength=((self.config["num_classes"] + 1)))
#bincounts, bincounts_idx = torch.sort(bincounts)
sample_targets_single = torch.argmax(
sample_targets.type(dtype=torch.long), axis=0)
#bincounts = torch.bincount(sample_predictions[:, ws].type(dtype=torch.long),
# minlength=(self.config["num_classes"] + 1))
#bincounts, bincounts_idx = torch.sort(bincounts)
sample_predictions_single = torch.argmax(
sample_predictions.type(dtype=torch.long), axis=0)
if target_files == 0:
sample_targets_single_files = sample_targets_single
else:
sample_targets_single_files = torch.cat(
(sample_targets_single_files, sample_targets_single), dim=0)
if target_files == 0:
sample_predictions_single_files = sample_predictions_single
else:
sample_predictions_single_files = torch.cat(
(sample_predictions_single_files, sample_predictions_single),
dim=0)
#logging.info(" Network_User: Segmentation: "
# "size of sequence labels {}".format(sample_targets_single_files.size()))
#del sample_targets_single, sample_predictions_single
#del targets_per_file, predictions_per_file
return sample_targets_single_files, sample_predictions_single_files
Z1 = prune_adj(Z1,non_zero_idx,args.ratio_graph) - id1
U1 = U1 + (adj1 - id1 - Z1)
Z2 = adj2 - id2 + U2
Z2 = prune_adj(Z2,non_zero_idx,args.ratio_graph) - id2
U2 = U2 + (adj2 - id2 - Z2)
if cnt == args.draw:
break
adj1,adj2 = adjs_to_diff[0], adjs_to_diff[1]
adj1 = prune_adj(adj1 - id1, non_zero_idx, args.ratio_graph)
adj2 = prune_adj(adj2 - id2, non_zero_idx, args.ratio_graph)
edge_index1, e_id1, _ = adjs[0]
edge_index2, e_id2, _ = adjs[1]
global_values[e_id1] = adj1[torch.transpose(edge_index1,0,1)]
global_values[e_id2] = adj2[torch.transpose(edge_index2,0,1)]
# print("Optimization Finished!")
train_acc, val_acc, tmp_test_acc = test(model, data)
log = 'After tune results: Ratio: {:d}, Train: {:.4f}, Val: {:.4f}, Test: {:.4f}'
# print(log.format(args.ratio, train_acc, val_acc, tmp_test_acc))
log_4_test = 'Tune Ratio: {:d}'
# print(log_4_test.format(args.ratio))
cur_adj1 = model.adj1.cpu().numpy()
cur_adj2 = model.adj2.cpu().numpy()
print("finish L1 training, num of edges * 2 + diag in adj1:", np.count_nonzero(cur_adj1))
# print("finish L1 training, num of edges * 2 + diag in adj2:", np.count_nonzero(cur_adj2))
# print("symmetry result adj1: ", testsymmetry(cur_adj1))
# print("symmetry result adj2: ", testsymmetry(cur_adj2))
# print("is equal of two adj", isequal(cur_adj1, cur_adj2))
def main():
#load encoder
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
#get data and encode it
sentences = []
labels = []
with open("./glue_data/SST-2/dev.tsv") as tsvfile:
tsvreader = csv.reader(tsvfile, delimiter="\t")
for i, line in enumerate(tsvreader):
if i > 0:
sentences += [line[0]]
labels += [int(line[1])]
# ici juste un tour pour voir quelle est la taille max . on visera un peu pls haut par sécurité.
max_len = 0
# For every sentence...
for sent in sentences:
# Tokenize the text and add `[CLS]` and `[SEP]` tokens.
input_ids = tokenizer.encode(sent, add_special_tokens=True)
# Update the maximum sentence length.
max_len = max(max_len, len(input_ids))
print('Max sentence length: ', max_len)
input_ids = []
for sent in sentences:
encoded_dict = tokenizer.encode(
sent, # Sentence to encode.
add_special_tokens=True, # Add '[CLS]' and '[SEP]'
max_length=max_len + 10, # Pad & truncate all sentences.
truncation=True,
pad_to_max_length=True,
return_tensors='pt', # Return pytorch tensors.
)
# Add the encoded sentence to the list.
input_ids.append(encoded_dict)
# Convert the lists into tensors.
input_ids = torch.cat(input_ids, dim=0)
labels = torch.tensor(labels)
# Print sentence 0, now as a list of IDs.
#print('Original: ', sentences[0])
#print('Token IDs:', input_ids[0])
# If there's a GPU available...
if torch.cuda.is_available():
# Tell PyTorch to use the GPU.
device = torch.device("cuda")
print('There are %d GPU(s) available.' % torch.cuda.device_count())
print('We will use the GPU:', torch.cuda.get_device_name(0))
# If not...
else:
print('No GPU available, using the CPU instead.')
device = torch.device("cpu")
# Load BertForSequenceClassification, the pretrained BERT model with a single
# linear classification layer on top.
model = RobertaForSequenceClassification.from_pretrained(
"./my_pretrained", # Use the 12-layer BERT model, with an uncased vocab. #please rather use "roberta-base"
num_labels=
2, # The number of output labels--2 for binary classification.
# You can increase this for multi-class tasks.
output_attentions=False, # Whether the model returns attentions weights.
output_hidden_states=
False, # Whether the model returns all hidden-states.
)
# If there's a GPU available...
if torch.cuda.is_available():
# Tell pytorch to run this model on the GPU.
model.cuda()
map_location = lambda storage, loc: storage.cuda()
else:
map_location = 'cpu'
#load saved model (which is finetuned roberta)
model.load_state_dict(
torch.load('./roberta_finetuned.pt', map_location=map_location))
model.eval()
#define the two forward functions (we cut our model in two parts)
def predict1(x): #ex: x = input_ids[0].unsqueeze(0).to(device)
emb = list(model.roberta.embeddings.children())[:1][0](
x) #embedding of x
return emb
def predict2(x, emb):
#some model requirements for the calculation
padding_idx = 1
input_shape = x.size()
seq_length = input_shape[1]
position_ids = create_position_ids_from_input_ids(x.to('cpu'),
1).to(device)
token_type_ids = torch.zeros(input_shape,
dtype=torch.long,
device=device)
#model calculations:
emb2 = list(model.roberta.embeddings.children())[1:][0](position_ids)
emb3 = list(model.roberta.embeddings.children())[1:][1](token_type_ids)
ess = list(model.roberta.embeddings.children())[1:][2](emb + emb2 +
emb3)
out_1st = list(model.roberta.embeddings.children())[1:][3](
ess) #result of the whole embedding layer of roberta
#getting result of encoder layer of roberta
out_2nd = model.roberta.encoder.layer[:12][0](out_1st)
for i in range(1, 12):
out_2nd = model.roberta.encoder.layer[:12][i](out_2nd[0])
#getting result of pooler layer of roberta
out_3nd = model.roberta.pooler(out_2nd[0])
out_4nd = (
out_2nd[0],
out_3nd,
) + out_2nd[1:]
out_fin = out_4nd[0]
#getting result of classifier layer of roberta
out = model.classifier(out_fin) #this is equivalent to model(x)
return out
#find numb neighboors of embedd among the embedding dictionary
def neighboors(embedd, numb):
emb_matrix = model.roberta.embeddings.word_embeddings.weight
normed_emb_matrix = F.normalize(emb_matrix, p=2, dim=1)
normed_emb_word = F.normalize(embedd, p=2, dim=0)
cosine_similarity = torch.matmul(
normed_emb_word, torch.transpose(normed_emb_matrix, 0, 1))
calc, closest_words = torch.topk(cosine_similarity, numb, dim=0)
print(tokenizer.decode(closest_words))
return closest_words
#find numb neighboors of embedd among candidates
def neighboors_np_dens_cand(embedd, rayon, candidates):
normed_emb_word = F.normalize(embedd, p=2, dim=0)
cosine_similarity = torch.matmul(normed_emb_word,
torch.transpose(candidates, 0, 1))
calc, closest_words = torch.topk(cosine_similarity, 1, dim=0)
compteur = 0
if rayon < 1.:
for t in range(len(cosine_similarity)):
if cosine_similarity[t] > rayon:
compteur += 1 #calculate the density around embedd, among all possible candidates
return closest_words, compteur
def perturb_iterative_fool_many(xvar,
embvar,
indlistvar,
yvar,
predict,
nb_iter,
eps,
epscand,
eps_iter,
loss_fn,
rayon,
delta_init=None,
minimize=False,
ord=np.inf,
clip_min=0.0,
clip_max=1.0,
l1_sparsity=None):
"""
Iteratively maximize the loss over the input. It is a shared method for
iterative attacks including IterativeGradientSign, LinfPGD, etc.
:param xvar: input data.
:param yvar: input labels.
:param predict: forward pass function.
:param nb_iter: number of iterations.
:param eps: maximum distortion.
:param eps_iter: attack step size.
:param loss_fn: loss function.
:param delta_init: (optional) tensor contains the random initialization.
:param minimize: (optional bool) whether to minimize or maximize the loss.
:param ord: (optional) the order of maximum distortion (inf or 2).
:param clip_min: mininum value per input dimension.
:param clip_max: maximum value per input dimension.
:param l1_sparsity: sparsity value for L1 projection.
- if None, then perform regular L1 projection.
- if float value, then perform sparse L1 descent from
Algorithm 1 in https://arxiv.org/pdf/1904.13000v1.pdf
:return: tensor containing the perturbed input.
"""
#will contain all words encountered during PGD
nb = len(indlistvar)
tablist = []
for t in range(nb):
tablist += [[]]
fool = False
#contain each loss on embed and each difference of loss on word nearest neighboor
loss_memory = np.zeros((nb_iter, ))
word_balance_memory = np.zeros((nb_iter, ))
candid = [torch.empty(0)] * nb
convers = [[]] * nb
for u in range(nb):
#prepare all potential candidates, once and for all
candidates = torch.empty([0, 768]).to(device)
conversion = []
emb_matrix = model.roberta.embeddings.word_embeddings.weight
normed_emb_matrix = F.normalize(emb_matrix, p=2, dim=1)
normed_emb_word = F.normalize(embvar[0][indlistvar[u]], p=2, dim=0)
cosine_similarity = torch.matmul(
normed_emb_word, torch.transpose(normed_emb_matrix, 0, 1))
for t in range(
len(cosine_similarity)): #evitez de faire DEUX boucles .
if cosine_similarity[t] > epscand:
if levenshtein(
tokenizer.decode(
torch.tensor([xvar[0][indlistvar[u]]])),
tokenizer.decode(torch.tensor([t]))) != 1:
candidates = torch.cat(
(candidates, normed_emb_matrix[t].unsqueeze(0)), 0)
conversion += [t]
candid[u] = candidates
convers[u] = conversion
print("nb of candidates :")
print(len(conversion))
#U, S, V = torch.svd(model.roberta.embeddings.word_embeddings.weight)
if delta_init is not None:
delta = delta_init
else:
delta = torch.zeros_like(embvar)
#PGD
delta.requires_grad_()
ii = 0
while ii < nb_iter and not (fool):
outputs = predict(xvar, embvar + delta)
loss = loss_fn(outputs, yvar)
if minimize:
loss = -loss
loss.backward()
if ord == np.inf:
grad_sign = delta.grad.data.sign()
grad_sign = tozerolist(grad_sign, indlistvar)
delta.data = delta.data + batch_multiply(eps_iter, grad_sign)
delta.data = batch_clamp(eps, delta.data)
delta.data = clamp(
embvar.data + delta.data,
clip_min,
clip_max #à retirer?
) - embvar.data
with torch.no_grad():
delta.data = tozero(delta.data, indlistvar)
if (ii % 300) == 0:
adverslist = []
for t in range(nb):
advers, nb_vois = neighboors_np_dens_cand(
(embvar + delta)[0][indlistvar[t]], rayon,
candid[t])
advers = int(advers[0])
advers = torch.tensor(convers[t][advers])
if len(tablist[t]) == 0:
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
elif not (first(
tablist[t][-1]) == tokenizer.decode(
advers.unsqueeze(0))):
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
adverslist += [advers]
word_balance_memory[ii] = float(
model(replacelist(xvar, indlistvar, adverslist),
labels=1 - yvar)[0]) - float(
model(replacelist(
xvar, indlistvar, adverslist),
labels=yvar)[0])
if word_balance_memory[ii] < 0:
fool = True
elif ord == 0:
grad = delta.grad.data
grad = tozero(grad, indlistvar)
grad = torch.matmul(
torch.cat((torch.matmul(grad, v)[:, :, :50],
torch.zeros([768 - 50]).to(device)), 2), v.t())
delta.data = delta.data + batch_multiply(eps_iter, grad)
delta.data[0] = my_proj_all(embvar.data[0] + delta.data[0],
embvar[0], indlistvar,
eps) - embvar.data[0]
delta.data = clamp(embvar.data + delta.data, clip_min,
clip_max) - embvar.data #à virer je pense
with torch.no_grad():
delta.data = tozero(delta.data, indlistvar)
if (ii % 300) == 0:
adverslist = []
for t in range(nb):
advers, nb_vois = neighboors_np_dens_cand(
(embvar + delta)[0][indlistvar[t]], rayon,
candid[t])
advers = int(advers[0])
advers = torch.tensor(convers[t][advers])
if len(tablist[t]) == 0:
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
elif not (first(
tablist[t][-1]) == tokenizer.decode(
advers.unsqueeze(0))):
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
adverslist += [advers]
word_balance_memory[ii] = float(
model(replacelist(xvar, indlistvar, adverslist),
labels=1 - yvar)[0]) - float(
model(replacelist(
xvar, indlistvar, adverslist),
labels=yvar)[0])
if word_balance_memory[ii] < 0:
fool = True
elif ord == 2:
grad = delta.grad.data
grad = tozero(grad, indlistvar)
grad = normalize_by_pnorm(grad)
delta.data = delta.data + batch_multiply(eps_iter, grad)
delta.data = clamp(embvar.data + delta.data, clip_min,
clip_max) - embvar.data
if eps is not None:
delta.data = clamp_by_pnorm(delta.data, ord, eps)
with torch.no_grad():
delta.data = tozero(delta.data, indlistvar)
if (ii % 300) == 0:
adverslist = []
for t in range(nb):
advers, nb_vois = neighboors_np_dens_cand(
(embvar + delta)[0][indlistvar[t]], rayon,
candid[t])
advers = int(advers[0])
advers = torch.tensor(convers[t][advers])
if len(tablist[t]) == 0:
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
elif not (first(
tablist[t][-1]) == tokenizer.decode(
advers.unsqueeze(0))):
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
adverslist += [advers]
word_balance_memory[ii] = float(
model(replacelist(xvar, indlistvar, adverslist),
labels=1 - yvar)[0]) - float(
model(replacelist(
xvar, indlistvar, adverslist),
labels=yvar)[0])
if word_balance_memory[ii] < 0:
fool = True
elif ord == 1:
grad = delta.grad.data
grad_sign = tozero(grad_sign, indvar)
abs_grad = torch.abs(grad)
batch_size = grad.size(0)
view = abs_grad.view(batch_size, -1)
view_size = view.size(1)
if l1_sparsity is None:
vals, idx = view.topk(1)
else:
vals, idx = view.topk(
int(np.round((1 - l1_sparsity) * view_size)))
out = torch.zeros_like(view).scatter_(1, idx, vals)
out = out.view_as(grad)
grad = grad.sign() * (out > 0).float()
grad = normalize_by_pnorm(grad, p=1)
delta.data = delta.data + batch_multiply(eps_iter, grad)
delta.data = batch_l1_proj(delta.data.cpu(), eps)
if embvar.is_cuda:
delta.data = delta.data.cuda()
delta.data = clamp(embvar.data + delta.data, clip_min,
clip_max) - embvar.data
else:
error = "Only ord = inf, ord = 1 and ord = 2 have been implemented"
raise NotImplementedError(error)
delta.grad.data.zero_()
with torch.no_grad():
loss_memory[ii] = loss
ii += 1
#plt.plot(loss_memory)
#plt.title("evolution of embed loss")
#plt.show()
#plt.plot(word_balance_memory)
#plt.title("evolution of word loss difference")
#plt.show()
emb_adv = clamp(embvar + delta, clip_min, clip_max)
return emb_adv, word_balance_memory, loss_memory, tablist, fool
#pgd attack classes
class PGDAttack(Attack, LabelMixin):
"""
The projected gradient descent attack (Madry et al, 2017).
The attack performs nb_iter steps of size eps_iter, while always staying
within eps from the initial point.
Paper: https://arxiv.org/pdf/1706.06083.pdf
:param predict: forward pass function.
:param loss_fn: loss function.
:param eps: maximum distortion.
:param nb_iter: number of iterations.
:param eps_iter: attack step size.
:param rand_init: (optional bool) random initialization.
:param clip_min: mininum value per input dimension.
:param clip_max: maximum value per input dimension.
:param ord: (optional) the order of maximum distortion (inf or 2).
:param targeted: if the attack is targeted.
"""
def __init__(self,
predict,
loss_fn=None,
eps=0.3,
epscand=0.03,
nb_iter=40,
eps_iter=0.01,
rayon=0.5,
rand_init=True,
clip_min=0.,
clip_max=1.,
ord=np.inf,
l1_sparsity=None,
targeted=False):
"""
Create an instance of the PGDAttack.
"""
super(PGDAttack, self).__init__(predict, loss_fn, clip_min,
clip_max)
self.eps = eps
self.epscand = epscand
self.nb_iter = nb_iter
self.eps_iter = eps_iter
self.rayon = rayon
self.rand_init = rand_init
self.ord = ord
self.targeted = targeted
if self.loss_fn is None:
self.loss_fn = nn.CrossEntropyLoss(
reduction="sum") #or no reduction
self.l1_sparsity = l1_sparsity
assert is_float_or_torch_tensor(self.eps_iter)
assert is_float_or_torch_tensor(self.eps)
def perturb_fool_many(self,
x,
emb,
indlist,
y=None): #list of ind of words to be perturbed
"""
Given examples (x, y), returns their adversarial counterparts with
an attack length of eps.
:param x: input tensor.
:param y: label tensor.
- if None and self.targeted=False, compute y as predicted
labels.
- if self.targeted=True, then y must be the targeted labels.
:return: tensor containing perturbed inputs.
"""
emb, y = self._verify_and_process_inputs(emb, y) #???
delta = torch.zeros_like(emb)
delta = nn.Parameter(delta)
if self.rand_init:
rand_init_delta(delta, emb, np.inf, self.eps, self.clip_min,
self.clip_max)
delta.data = clamp(emb + delta.data,
min=self.clip_min,
max=self.clip_max) - emb
with torch.no_grad():
for t in range(delta.size()[1]):
if not (t in indlist):
for k in range(delta.size()[2]):
delta[0][t][k] = 0
if self.ord == 0:
delta[0] = my_proj_all(emb[0] + delta[0], emb[0], indlist,
self.eps) - emb[0]
rval, word_balance_memory, loss_memory, tablist, fool = perturb_iterative_fool_many(
x,
emb,
indlist,
y,
self.predict,
nb_iter=self.nb_iter,
eps=self.eps,
epscand=self.epscand,
eps_iter=self.eps_iter,
loss_fn=self.loss_fn,
minimize=self.targeted,
ord=self.ord,
clip_min=self.clip_min,
clip_max=self.clip_max,
delta_init=delta,
l1_sparsity=self.l1_sparsity,
rayon=self.rayon)
return rval.data, word_balance_memory, loss_memory, tablist, fool
###here we actually attack all sentences
res_se = []
res_or = []
res_lw = []
res_lg = []
res_ne = []
res_cs = []
# l1 : list of all the sentences we will attack
l1 = [0, 1, 3, 6, 150, 173, 197]
for iid in l1:
for eps_iter in [0.5]:
eps = 0.7 #embeddings distance with norm ord
epscand = 0.25 #fix nb of candidates according to cosim
nb_iter = 8001
ord = np.inf #norm choice
rayon = 1. #density search
t0 = time()
print("\n")
mf = 0
x = input_ids[iid].unsqueeze(0).to(device)
y = labels[iid].unsqueeze(0).to(device)
neigh = 3
indlist = range(1, sentlong(x[0]))
nind = len(indlist)
if float(model(x, labels=y)[0]) > float(model(
x, labels=1 - y)[0]): #if model misclassifies
mf += 1
print("sentence already misclassified")
#return res_se, res_or,res_lw,res_lg,res_ne,res_cs
else: #model classifies correctly so it makes sense to try to fool it
print("original sentence:")
print(tokenizer.decode(x[0]))
print("is classified as:")
if bool(y == 1):
print("positive")
else:
print("negative")
print("\n")
orig_wordlist = []
for u in range(nind):
#print("let's change word number"+str(indlist[u])+"which is:")
orig_word = tokenizer.decode(x[0][indlist[u]].unsqueeze(0))
#print(orig_word)
orig_wordlist += [orig_word]
#print("\n")
emb = predict1(x)
new_word = [''] * nind
print("Does our PGD output's first neighboor fool model?:")
att = PGDAttack(predict2,
loss_fn=None,
eps=eps,
epscand=epscand,
nb_iter=nb_iter,
eps_iter=eps_iter,
rayon=rayon,
rand_init=True,
clip_min=-1.,
clip_max=1.,
ord=ord,
l1_sparsity=None,
targeted=False)
rval, word_balance_memory, loss_memory, tablist, fool = att.perturb_fool_many(
x, emb, indlist, y)
print(fool)
csnlist = [0] * nind
for u in range(nind):
new_word[u] = first(tablist[u][-1])
print("\n")
for u in range(nind):
csnlist[u] = float(
torch.matmul(
F.normalize(
model.roberta.embeddings.word_embeddings(
torch.tensor(
tokenizer.encode(
new_word[u])[1]).to(device)),
p=2,
dim=0),
torch.transpose(
F.normalize(
model.roberta.embeddings.word_embeddings(
x[0][indlist[u]]).unsqueeze(0),
p=2,
dim=1), 0, 1)))
print(tokenizer.decode(x[0]))
print(orig_wordlist)
print(tablist)
print(lenlist(tablist))
print(new_word)
print(csnlist)
if fool:
res_se += [tokenizer.decode(x[0])]
res_or += [orig_wordlist]
res_lw += [tablist]
res_lg += [lenlist(tablist)]
res_ne += [new_word]
res_cs += [csnlist]
t1 = time()
print('function takes %f' % (t1 - t0))
df = pd.DataFrame(list(zip(res_se, res_or, res_lw, res_lg, res_ne,
res_cs)),
columns=[
'sentence', 'original word',
'not-fooling words ( = path)', 'path length',
'new word', 'csn similarity'
])
df.to_csv('results/results.csv', index=False) #r
def perturb_iterative_fool_many(xvar,
embvar,
indlistvar,
yvar,
predict,
nb_iter,
eps,
epscand,
eps_iter,
loss_fn,
rayon,
delta_init=None,
minimize=False,
ord=np.inf,
clip_min=0.0,
clip_max=1.0,
l1_sparsity=None):
"""
Iteratively maximize the loss over the input. It is a shared method for
iterative attacks including IterativeGradientSign, LinfPGD, etc.
:param xvar: input data.
:param yvar: input labels.
:param predict: forward pass function.
:param nb_iter: number of iterations.
:param eps: maximum distortion.
:param eps_iter: attack step size.
:param loss_fn: loss function.
:param delta_init: (optional) tensor contains the random initialization.
:param minimize: (optional bool) whether to minimize or maximize the loss.
:param ord: (optional) the order of maximum distortion (inf or 2).
:param clip_min: mininum value per input dimension.
:param clip_max: maximum value per input dimension.
:param l1_sparsity: sparsity value for L1 projection.
- if None, then perform regular L1 projection.
- if float value, then perform sparse L1 descent from
Algorithm 1 in https://arxiv.org/pdf/1904.13000v1.pdf
:return: tensor containing the perturbed input.
"""
#will contain all words encountered during PGD
nb = len(indlistvar)
tablist = []
for t in range(nb):
tablist += [[]]
fool = False
#contain each loss on embed and each difference of loss on word nearest neighboor
loss_memory = np.zeros((nb_iter, ))
word_balance_memory = np.zeros((nb_iter, ))
candid = [torch.empty(0)] * nb
convers = [[]] * nb
for u in range(nb):
#prepare all potential candidates, once and for all
candidates = torch.empty([0, 768]).to(device)
conversion = []
emb_matrix = model.roberta.embeddings.word_embeddings.weight
normed_emb_matrix = F.normalize(emb_matrix, p=2, dim=1)
normed_emb_word = F.normalize(embvar[0][indlistvar[u]], p=2, dim=0)
cosine_similarity = torch.matmul(
normed_emb_word, torch.transpose(normed_emb_matrix, 0, 1))
for t in range(
len(cosine_similarity)): #evitez de faire DEUX boucles .
if cosine_similarity[t] > epscand:
if levenshtein(
tokenizer.decode(
torch.tensor([xvar[0][indlistvar[u]]])),
tokenizer.decode(torch.tensor([t]))) != 1:
candidates = torch.cat(
(candidates, normed_emb_matrix[t].unsqueeze(0)), 0)
conversion += [t]
candid[u] = candidates
convers[u] = conversion
print("nb of candidates :")
print(len(conversion))
#U, S, V = torch.svd(model.roberta.embeddings.word_embeddings.weight)
if delta_init is not None:
delta = delta_init
else:
delta = torch.zeros_like(embvar)
#PGD
delta.requires_grad_()
ii = 0
while ii < nb_iter and not (fool):
outputs = predict(xvar, embvar + delta)
loss = loss_fn(outputs, yvar)
if minimize:
loss = -loss
loss.backward()
if ord == np.inf:
grad_sign = delta.grad.data.sign()
grad_sign = tozerolist(grad_sign, indlistvar)
delta.data = delta.data + batch_multiply(eps_iter, grad_sign)
delta.data = batch_clamp(eps, delta.data)
delta.data = clamp(
embvar.data + delta.data,
clip_min,
clip_max #à retirer?
) - embvar.data
with torch.no_grad():
delta.data = tozero(delta.data, indlistvar)
if (ii % 300) == 0:
adverslist = []
for t in range(nb):
advers, nb_vois = neighboors_np_dens_cand(
(embvar + delta)[0][indlistvar[t]], rayon,
candid[t])
advers = int(advers[0])
advers = torch.tensor(convers[t][advers])
if len(tablist[t]) == 0:
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
elif not (first(
tablist[t][-1]) == tokenizer.decode(
advers.unsqueeze(0))):
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
adverslist += [advers]
word_balance_memory[ii] = float(
model(replacelist(xvar, indlistvar, adverslist),
labels=1 - yvar)[0]) - float(
model(replacelist(
xvar, indlistvar, adverslist),
labels=yvar)[0])
if word_balance_memory[ii] < 0:
fool = True
elif ord == 0:
grad = delta.grad.data
grad = tozero(grad, indlistvar)
grad = torch.matmul(
torch.cat((torch.matmul(grad, v)[:, :, :50],
torch.zeros([768 - 50]).to(device)), 2), v.t())
delta.data = delta.data + batch_multiply(eps_iter, grad)
delta.data[0] = my_proj_all(embvar.data[0] + delta.data[0],
embvar[0], indlistvar,
eps) - embvar.data[0]
delta.data = clamp(embvar.data + delta.data, clip_min,
clip_max) - embvar.data #à virer je pense
with torch.no_grad():
delta.data = tozero(delta.data, indlistvar)
if (ii % 300) == 0:
adverslist = []
for t in range(nb):
advers, nb_vois = neighboors_np_dens_cand(
(embvar + delta)[0][indlistvar[t]], rayon,
candid[t])
advers = int(advers[0])
advers = torch.tensor(convers[t][advers])
if len(tablist[t]) == 0:
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
elif not (first(
tablist[t][-1]) == tokenizer.decode(
advers.unsqueeze(0))):
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
adverslist += [advers]
word_balance_memory[ii] = float(
model(replacelist(xvar, indlistvar, adverslist),
labels=1 - yvar)[0]) - float(
model(replacelist(
xvar, indlistvar, adverslist),
labels=yvar)[0])
if word_balance_memory[ii] < 0:
fool = True
elif ord == 2:
grad = delta.grad.data
grad = tozero(grad, indlistvar)
grad = normalize_by_pnorm(grad)
delta.data = delta.data + batch_multiply(eps_iter, grad)
delta.data = clamp(embvar.data + delta.data, clip_min,
clip_max) - embvar.data
if eps is not None:
delta.data = clamp_by_pnorm(delta.data, ord, eps)
with torch.no_grad():
delta.data = tozero(delta.data, indlistvar)
if (ii % 300) == 0:
adverslist = []
for t in range(nb):
advers, nb_vois = neighboors_np_dens_cand(
(embvar + delta)[0][indlistvar[t]], rayon,
candid[t])
advers = int(advers[0])
advers = torch.tensor(convers[t][advers])
if len(tablist[t]) == 0:
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
elif not (first(
tablist[t][-1]) == tokenizer.decode(
advers.unsqueeze(0))):
tablist[t] += [
(tokenizer.decode(advers.unsqueeze(0)), ii,
nb_vois)
]
adverslist += [advers]
word_balance_memory[ii] = float(
model(replacelist(xvar, indlistvar, adverslist),
labels=1 - yvar)[0]) - float(
model(replacelist(
xvar, indlistvar, adverslist),
labels=yvar)[0])
if word_balance_memory[ii] < 0:
fool = True
elif ord == 1:
grad = delta.grad.data
grad_sign = tozero(grad_sign, indvar)
abs_grad = torch.abs(grad)
batch_size = grad.size(0)
view = abs_grad.view(batch_size, -1)
view_size = view.size(1)
if l1_sparsity is None:
vals, idx = view.topk(1)
else:
vals, idx = view.topk(
int(np.round((1 - l1_sparsity) * view_size)))
out = torch.zeros_like(view).scatter_(1, idx, vals)
out = out.view_as(grad)
grad = grad.sign() * (out > 0).float()
grad = normalize_by_pnorm(grad, p=1)
delta.data = delta.data + batch_multiply(eps_iter, grad)
delta.data = batch_l1_proj(delta.data.cpu(), eps)
if embvar.is_cuda:
delta.data = delta.data.cuda()
delta.data = clamp(embvar.data + delta.data, clip_min,
clip_max) - embvar.data
else:
error = "Only ord = inf, ord = 1 and ord = 2 have been implemented"
raise NotImplementedError(error)
delta.grad.data.zero_()
with torch.no_grad():
loss_memory[ii] = loss
ii += 1
#plt.plot(loss_memory)
#plt.title("evolution of embed loss")
#plt.show()
#plt.plot(word_balance_memory)
#plt.title("evolution of word loss difference")
#plt.show()
emb_adv = clamp(embvar + delta, clip_min, clip_max)
return emb_adv, word_balance_memory, loss_memory, tablist, fool
def forward(input, filter):
"""
Compute Winograd convolution.
:param input:
:param filter:
:return: output
"""
N, C, H, W = input.size()
K, Cprime, r, rprime = filter.size()
assert H == W
assert r == rprime
assert C == Cprime
m = 2
a = m + r - 1
# TODO pad with zeros the input for perfect tiling and slice the output.
overlap = r - 1
if (H >= 4 and H % 2 == 0) is False:
raise Exception("Only input for perfect tiling is supported.")
input = torch.transpose(input, 0, 1)
assert input.size() == (C, N, H, W)
# ntile = int(math.ceil(H//a))
# P = N * ntile * ntile
T = (W - a) // overlap + 1 # tiles_per_channel
P = N * T * T
U = torch.zeros(K, C, a, a)
V = torch.zeros(C, P, a, a)
for k in range(K):
for c in range(C):
U[k,
c] = torch.matmul(Winograd.G,
torch.matmul(filter[k, c], Winograd.G_T))
for n in range(N):
for tH in range(T):
for tW in range(T):
for c in range(C):
b = n * (T * T) + tH * T + tW
vH = tH * (r - 1)
vW = tW * (r - 1)
V[c, b] = torch.matmul(
Winograd.B_T,
torch.matmul(input[c, n, vH:vH + a, vW:vW + a],
Winograd.B))
M = torch.zeros(K, P, a, a)
for k in range(K):
for b in range(P):
for c in range(C):
M[k, b] += U[k, c] * V[c, b]
# M = torch.matmul(U, V)
out_size = H - r + 1
Y = torch.zeros(K, N, out_size, out_size)
for k in range(K):
for n in range(N):
for tH in range(T):
for tW in range(T):
b = n * (T * T) + tH * T + tW
oH = tH * m
oW = tW * m
Y[k, n, oH:oH + m, oW:oW + m] = torch.matmul(
Winograd.A_T, torch.matmul(M[k, b], Winograd.A))
Y = torch.transpose(Y, 0, 1)
return Y
