def forward(self, x):
x, indices = F.adaptive_max_pool1d(x,
output_size=7,
return_indices=True)
x = F.adaptive_max_pool1d(x, output_size=1)
return x, indices
def forward(self, input: Tensor) -> Tensor:
input = self.quant_handle(input)
return F.adaptive_max_pool1d(input, self.output_size,
self.return_indices)
def adaptive_avgmax_pool1d(x, output_size=1):
x_avg = F.adaptive_avg_pool1d(x, output_size)
x_max = F.adaptive_max_pool1d(x, output_size)
return 0.5 * (x_avg + x_max)
def _forward_features(self, x): for l in self.conv: x = F.relu(l(x)) x = F.adaptive_max_pool1d(x, output_size=1) return x
def forward(self, x):
batch_size = x.size(0)
x = x.repeat(1, 1, 8)
#print(x.shape)
# shape of x : batch, feature, npoints, neighbors
# =>
# transformer(shared wq, wk, wv to xi)
i = 1
points = x
tic = time.time()
tic_init = tic
i, tic = memchk(i, string="points", tic=tic)
x, abs_x, idx1 = get_neighbors(x, k=self.k) # b, 64, 1024, 20
i, tic = memchk(i, string="kNN1", tic=tic)
x1 = self.conv1(x, abs_x, idx1, points) # b, 64, 1024
i, tic = memchk(i, string="conv1", tic=tic)
x1 = self.act1(self.bn1(x1)).squeeze(3)
i, tic = memchk(i, string="bnrelu1", tic=tic)
x, abs_x, idx2 = get_neighbors(x1, k=self.k) # b, 64, 1024, 20
i, tic = memchk(i, string="kNN2", tic=tic)
x2 = self.conv2(x, abs_x, idx2, points) # b, 64, 1024
i, tic = memchk(i, string="conv2", tic=tic)
x2 = self.act2(self.bn2(x2)).squeeze(3)
i, tic = memchk(i, string="bnrelu2", tic=tic)
x, abs_x, idx3 = get_neighbors(x2, k=self.k) # b, 64, 1024, 20
i, tic = memchk(i, string="kNN3", tic=tic)
x3 = self.conv3(x, abs_x, idx3, points) # b, 128, 1024
i, tic = memchk(i, string="conv3", tic=tic)
x3 = self.act3(self.bn3(x3)).squeeze(3)
i, tic = memchk(i, string="bnrelu3", tic=tic)
x, abs_x, idx4 = get_neighbors(x3, k=self.k) # b, 64, 1024, 20
i, tic = memchk(i, string="kNN4", tic=tic)
x4 = self.conv4(x, abs_x, idx4, points) # b, 256, 1024, 20
i, tic = memchk(i, string="conv4", tic=tic)
x4 = self.act4(self.bn4(x4)).squeeze(3)
i, tic = memchk(i, string="bnrelu4", tic=tic)
x = self.conv5(x4)
i, tic = memchk(i, string="conv5", tic=tic) # 8
x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1)
x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2)
x = self.dp1(x)
i, tic = memchk(i, string="dp1", tic=tic) # 8
x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2)
x = self.dp2(x)
i, tic = memchk(i, string="dp2", tic=tic) # 8
x = self.linear3(x)
i, tic_final = memchk(i, string="fc", tic=tic) # 8
print("Total elapsed time : {} ms".format(
round((tic_final - tic_init) * 1000, 3)))
return x
def forward(
self,
x_diag: torch.LongTensor,
x_lengths: torch.LongTensor,
x_timedels: torch.FloatTensor = None,
x_age: torch.FloatTensor = None,
x_gender: torch.LongTensor = None,
x_time_delta_mean: torch.FloatTensor = None,
all_input_zeros: bool = False,
) -> torch.Tensor:
batch_size, max_seq_length, _ = x_diag.shape
if all_input_zeros:
x_diag = torch.zeros_like(x_diag)
#### Embeddings
x_embed = self.diag_embed(x_diag)
## rNN Layers
# This does early fusion with feature replication
x_gender = (self.gender_embed(x_gender).unsqueeze(1).expand(
batch_size, max_seq_length, -1))
x_age = x_age.unsqueeze(1).expand(batch_size, max_seq_length, -1)
x_time_delta_mean = x_time_delta_mean.unsqueeze(1).expand(
batch_size, max_seq_length, -1)
if self.use_adder:
x_embed_sum = self.embed_sum(x_embed)
else:
x_embed_sum = self.embed_conv(x_embed.permute(0, 2, 1,
3)).squeeze(1)
# we replicate the scalar features across every time-step
x_embed_sum = torch.cat(
[
x_embed_sum, x_gender, x_age, x_time_delta_mean,
x_timedels.unsqueeze(2)
],
dim=2,
)
rnn_outs, _ = self.rnn(x_embed_sum)
rnn_outs = rnn_outs.contiguous().view(batch_size, max_seq_length,
self.rnn_hidden_size)
attn_out, _ = self.attn(rnn_outs, rnn_outs, rnn_outs)
rnn_attn_max = F.adaptive_max_pool1d(attn_out.permute(0, 2, 1),
1).squeeze()
rnn_out = gather_last_relevant_hidden(rnn_outs, x_lengths)
if self.use_mh_attn:
rnn_out = (1 - F.sigmoid(self.attn_alpha)
) * rnn_out + rnn_attn_max * F.sigmoid(self.attn_alpha)
z = self.fc_w_feats(rnn_out)
if self.do_bn:
z = self.ln(z)
z = self.dropout(self.activation(z))
y_pred = self.out(z)
y_out = F.relu(y_pred.view(-1))
return y_out
def forward(self, x, norm_plt, cls_label, gt=None):
B, C, N = x.size()
idx, _ = knn(
x, k=self.k
) # different with DGCNN, the knn search is only in 3D space
xyz = get_scorenet_input(x, k=self.k, idx=idx) # ScoreNet input
#################
# use MLP at the 1st layer, same with DGCNN
x = get_graph_feature(x, k=self.k, idx=idx)
x = x.permute(0, 3, 1, 2) # b,2cin,n,k
x = F.relu(self.conv1(x))
x1 = x.max(dim=-1, keepdim=False)[0]
#################
# replace the last 4 DGCNN-EdgeConv with PAConv:
"""CUDA implementation of PAConv: (presented in the supplementary material of the paper)"""
"""feature transformation:"""
x2, center2 = feat_trans_dgcnn(point_input=x1,
kernel=self.matrice2,
m=self.m2)
score2 = self.scorenet2(xyz, calc_scores=self.calc_scores, bias=0)
"""assemble with scores:"""
x = assemble_dgcnn(score=score2,
point_input=x2,
center_input=center2,
knn_idx=idx,
aggregate='sum')
x2 = F.relu(self.bn2(x))
x3, center3 = feat_trans_dgcnn(point_input=x2,
kernel=self.matrice3,
m=self.m3)
score3 = self.scorenet3(xyz, calc_scores=self.calc_scores, bias=0)
x = assemble_dgcnn(score=score3,
point_input=x3,
center_input=center3,
knn_idx=idx,
aggregate='sum')
x3 = F.relu(self.bn3(x))
x4, center4 = feat_trans_dgcnn(point_input=x3,
kernel=self.matrice4,
m=self.m4)
score4 = self.scorenet4(xyz, calc_scores=self.calc_scores, bias=0)
x = assemble_dgcnn(score=score4,
point_input=x4,
center_input=center4,
knn_idx=idx,
aggregate='sum')
x4 = F.relu(self.bn4(x))
x5, center5 = feat_trans_dgcnn(point_input=x4,
kernel=self.matrice5,
m=self.m5)
score5 = self.scorenet5(xyz, calc_scores=self.calc_scores, bias=0)
x = assemble_dgcnn(score=score5,
point_input=x5,
center_input=center5,
knn_idx=idx,
aggregate='sum')
x5 = F.relu(self.bn5(x))
###############
xx = torch.cat((x1, x2, x3, x4, x5), dim=1)
xc = F.relu(self.convt(xx))
xc = F.adaptive_max_pool1d(xc, 1).view(B, -1)
cls_label = cls_label.view(B, 16, 1)
cls_label = F.relu(self.convc(cls_label))
cls = torch.cat((xc.view(B, 1024, 1), cls_label), dim=1)
cls = cls.repeat(1, 1, N) # B,1088,N
x = torch.cat((xx, cls), dim=1) # 1088+64*3
x = F.relu(self.conv6(x))
x = self.dp1(x)
x = F.relu(self.conv7(x))
x = self.dp2(x)
x = F.relu(self.conv8(x))
x = self.conv9(x)
x = F.log_softmax(x, dim=1)
x = x.permute(0, 2, 1) # b,n,50
if gt is not None:
return x, F.nll_loss(x.contiguous().view(-1, self.num_part),
gt.view(-1, 1)[:, 0])
else:
return x
def forward(self, x, label=None, criterion=None):
B, C, N = x.size()
idx, _ = knn(
x, k=self.k
) # different with DGCNN, the knn search is only in 3D space
xyz = get_scorenet_input(
x, idx=idx, k=self.k
) # ScoreNet input: 3D coord difference concat with coord: b,6,n,k
##################
# replace all the DGCNN-EdgeConv with PAConv:
"""CUDA implementation of PAConv: (presented in the supplementary material of the paper)"""
"""feature transformation:"""
point1, center1 = feat_trans_dgcnn(point_input=x,
kernel=self.matrice1,
m=self.m1) # b,n,m1,o1
score1 = self.scorenet1(xyz, calc_scores=self.calc_scores, bias=0.5)
"""assemble with scores:"""
point1 = assemble_dgcnn(score=score1,
point_input=point1,
center_input=center1,
knn_idx=idx,
aggregate='sum') # b,o1,n
point1 = F.relu(self.bn1(point1))
point2, center2 = feat_trans_dgcnn(point_input=point1,
kernel=self.matrice2,
m=self.m2)
score2 = self.scorenet2(xyz, calc_scores=self.calc_scores, bias=0.5)
point2 = assemble_dgcnn(score=score2,
point_input=point2,
center_input=center2,
knn_idx=idx,
aggregate='sum')
point2 = F.relu(self.bn2(point2))
point3, center3 = feat_trans_dgcnn(point_input=point2,
kernel=self.matrice3,
m=self.m3)
score3 = self.scorenet3(xyz, calc_scores=self.calc_scores, bias=0.5)
point3 = assemble_dgcnn(score=score3,
point_input=point3,
center_input=center3,
knn_idx=idx,
aggregate='sum')
point3 = F.relu(self.bn3(point3))
point4, center4 = feat_trans_dgcnn(point_input=point3,
kernel=self.matrice4,
m=self.m4)
score4 = self.scorenet4(xyz, calc_scores=self.calc_scores, bias=0.5)
point4 = assemble_dgcnn(score=score4,
point_input=point4,
center_input=center4,
knn_idx=idx,
aggregate='sum')
point4 = F.relu(self.bn4(point4))
##################
point = torch.cat((point1, point2, point3, point4), dim=1)
point = F.relu(self.conv5(point))
point11 = F.adaptive_max_pool1d(point, 1).view(B, -1)
point22 = F.adaptive_avg_pool1d(point, 1).view(B, -1)
point = torch.cat((point11, point22), 1)
point = F.relu(self.bn11(self.linear1(point)))
point = self.dp1(point)
point = F.relu(self.bn22(self.linear2(point)))
point = self.dp2(point)
point = self.linear3(point)
if criterion is not None:
return point, criterion(point, label) # return output and loss
else:
return point
def forward(self, x, vars=None, bn_training=True):
if vars is None:
vars = self.vars
idx = 0
bn_idx = 0
for name, param in self.config:
if name is 'conv1d':
w, b = vars[idx], vars[idx + 1]
# remember to keep synchrozied of forward_encoder and forward_decoder!
x = F.conv1d(x, w, b, stride=param[3], padding=param[4])
idx += 2
if param[5] == 1:
x1 = x
elif param[5] == 2:
x2 = x
elif param[5] == 3:
x3 = x
elif name is 'concat':
x = torch.cat([x1, x2, x3], 1)
elif name is 'linear':
w, b = vars[idx], vars[idx + 1]
x = F.linear(x, w, b)
idx += 2
elif name is 'bn':
w, b = vars[idx], vars[idx + 1]
running_mean, running_var = self.vars_bn[bn_idx], self.vars_bn[
bn_idx + 1]
x = F.batch_norm(x,
running_mean,
running_var,
weight=w,
bias=b,
training=bn_training)
idx += 2
bn_idx += 2
elif name is 'flatten':
# print(x.shape)
x = x.view(x.size(0), -1)
elif name is 'relu':
x = F.relu(x, inplace=param[0])
elif name is 'max_pool1d':
x = F.max_pool1d(x, param[0])
if param[1] == 1:
x1 = x
elif name is 'SE':
x_se = F.adaptive_max_pool1d(x, param[0])
x_se = x_se.view(x_se.size(0), -1)
w1, b1 = vars[idx], vars[idx + 1]
x_se = F.linear(x_se, w1, b1)
idx += 2
x_se = F.relu(x_se, inplace=param[3])
w2, b2 = vars[idx], vars[idx + 1]
x_se = F.linear(x_se, w2, b2)
idx += 2
x_se = F.sigmoid(x_se).unsqueeze(2)
x_se = x_se.expand_as(x)
x = x * x_se
else:
raise NotImplementedError
assert idx == len(vars)
assert bn_idx == len(self.vars_bn)
return x
def forward(self, bins):
lst_encodings = []
bio_targets = []
ec_targets = []
event_duration_targets = []
independent_events_targets = []
ec_independent_targets = []
ec_independent_event_ids = []
for bin in bins:
bin_tweets = bin[0]
bin_tweets = bin_tweets.to('cuda')
bin_tweets = bin_tweets.squeeze(0)
bio_targets.extend(bin[1].data.tolist())
ec_targets.extend(bin[2])
event_duration_targets.extend(
bin[3]) #(0,1) with duration 0000111111111111110000
independent_events_targets.extend(
bin[4]) #(0,1) without duration 0000111110000111110000
ec_independent_targets.extend(
bin[5]) #(0,1,2,3,4) without duration 0000111110000222220000
ec_independent_event_ids.extend(
bin[6]
) #(-1,0,1,2,3,4) without duration -1-1-1-100000-1-1-1-111111-1-1-1-1
tweets_length = torch.stack(bin[7]).to('cuda')
tweets_length = tweets_length.squeeze(1)
bin_tweets, tweets_length = self.sort_batch(
bin_tweets, tweets_length)
embeds = self.word_embeddings(bin_tweets).permute(1, 0, 2)
if self.config.use_dropout == True:
embeds = self.dropoutEmbeddings(embeds)
embeds = pack_padded_sequence(embeds, tweets_length) # unpad
if self.config.tweet_representation == "embeddings":
tweet_representation = embeds
tweet_representation, _ = pad_packed_sequence(
tweet_representation)
elif self.config.tweet_representation == "lstm":
lstm_output, _ = self.lstm(embeds)
tweet_representation, _ = pad_packed_sequence(lstm_output)
if self.config.bin_representation == "avg":
tl_expanded = (tweets_length).unsqueeze(1)
avg_pool_byhand = torch.sum(
tweet_representation,
dim=0) / tl_expanded.to(dtype=torch.float)
bin_representation = avg_pool_byhand
elif self.config.bin_representation == "max":
max_pool_byhand = torch.cat([
torch.max(i[:l], dim=0)[0].view(1, -1) for i, l in zip(
tweet_representation.permute(1, 0, 2), tweets_length)
],
dim=0)
bin_representation = max_pool_byhand
bin_representation = bin_representation.view(
bin_representation.size(0), 1, bin_representation.size(1))
if self.config.bin_features == "avg":
avg_pool = F.adaptive_avg_pool1d(
bin_representation.permute(1, 2, 0), 1) #.view(1,1, -1)
avg_pool.squeeze_(2).unsqueeze_(0) # 1,1,32
bin_features = avg_pool
elif self.config.bin_features == "max":
max_pool = F.adaptive_max_pool1d(
bin_representation.permute(1, 2, 0), 1) #.view(1,1, -1)
max_pool.squeeze_(2).unsqueeze_(0) # 1,1,32
bin_features = max_pool
elif self.config.bin_features == "tweet_attention":
bin_representation.squeeze_(1)
scores = self.hidden2score(bin_representation)
normalized_scores = self.softmax(scores)
a_e = normalized_scores * bin_representation
attented_representation = torch.sum(a_e, dim=0)
bin_features = attented_representation #.view(1,1, -1)
bin_features.unsqueeze_(0).unsqueeze_(0)
elif self.config.bin_features == "tweet_attention2":
bin_representation.squeeze_(1)
inputs = bin_representation
scores = self.relu(inputs.matmul(self.attention_weights2))
scores = self.softmax3(scores)
weighted = torch.mul(inputs,
scores.unsqueeze(-1).expand_as(inputs))
representations = weighted.sum(0).squeeze()
bin_features = representations #.view(1, 1, -1)
bin_features.unsqueeze_(0).unsqueeze_(0)
elif self.config.bin_features == "cnn":
embedded = bin_representation.squeeze(1).unsqueeze_(
0).unsqueeze_(0)
conved = [
F.relu(conv(embedded)).squeeze(3) for conv in self.convs
]
if self.config.cnn_pool == "max":
pooled = [
F.max_pool1d(conv, conv.shape[2]).squeeze(2)
for conv in conved
]
elif self.config.cnn_pool == "avg":
pooled = [
F.avg_pool1d(conv, conv.shape[2]).squeeze(2)
for conv in conved
]
bin_features = torch.cat(pooled, dim=1)
if self.config.use_dropout == True:
bin_features = self.cnn_dropout(torch.cat(pooled, dim=1))
bin_features.unsqueeze_(0)
if self.config.non_linearity_bin_features == "relu":
bin_features = self.relu(bin_features)
elif self.config.non_linearity_bin_features == "tanh":
bin_features = self.tanh(bin_features)
lst_encodings.append(bin_features)
lst_encodings = torch.cat(lst_encodings) # .to(self.device)
if self.config.batch_norm == True:
lst_encodings = self.dense1_bn(
lst_encodings.view(1, lst_encodings.size(2),
lst_encodings.size(0)))
lst_encodings = lst_encodings.view(lst_encodings.size(2), 1,
lst_encodings.size(1))
if self.config.use_BIO_LSTM == True:
lstm_out2, _ = self.lstm2(lst_encodings)
if self.config.use_dropout == True:
lstm_out2 = self.dropoutLSTM2out(lstm_out2)
match_representation = lstm_out2
else:
match_representation = lst_encodings
match_representation = match_representation.squeeze(1)
tag_space = self.hidden2tag(match_representation)
return tag_space, bio_targets, ec_targets, event_duration_targets, independent_events_targets, ec_independent_targets, ec_independent_event_ids
def forward(self, input):
'''
010 110 110
|
conv layer
|
Lstm layer
|
reparameterization(optional)
INPUT: Batch_size x (V) extra process x Sentence_Num x Sentence_Max_Len
'''
# truncate kernel for saving resource
batch_size = input.shape[0]
sentence_num = input.shape[1]
if len(input.shape) > 2:
sentence_size = input.shape[2]
# else:
# sentence_num = 10 if sentence_num>10
# input = input.contiguous().view(batch_size, sentence_num, -1)
time_1 = time.time()
input, weight, compress_size = getattr(self,
'ver{}'.format(self.id))(input)
time_1 = time.time() - time_1
if self.use_RNN:
time_2 = time.time()
self.cnn_out = []
for i in range(self.cnn_num):
out = self.same_pad_conv2d(input, weight[i], self.bias[i],
self.stride[i])
out = self.relu(out)
# out = F.adaptive_max_pool2d(out, output_size = (sentence_num, 1)).squeeze(-1)
# out = out.permute(0,2,1) # output: batch, sentence_num, input size
self.cnn_out.append(out)
concat_out = torch.cat(self.cnn_out, dim=1).permute(0, 2, 3, 1)
word_att = self.word_attention(concat_out)
time_2 = time.time() - time_2
time_3 = time.time()
gru_out, h_n = self.gru(word_att)
# gru_out.unsqueeze(2)
sen_att = self.sen_attention(gru_out)
output = self.fc(sen_att).squeeze(-1)
time_3 = time.time() - time_3
self.tot_time += time_1 + time_2 + time_3
# print("1:{}, 2:{}, 3:{}, tot:{}".format(time_1/tot_time,time_2/tot_time,time_3/tot_time,tot_time))
else:
input = input.view(batch_size, compress_size, 1, -1)
self.cnn_out = []
for i in range(self.cnn_num):
out = self.F.conv2d(input, weight[i], self.bias[i]).squeeze(-2)
out = self.relu(out)
out = F.adaptive_max_pool1d(out, output_size=1).squeeze(-1)
self.cnn_out.append(out)
combi_out = torch.cat(self.cnn_out, dim=1)
output = self.fc1(combi_out).squeeze(-1)
return output
def forward(self, q_vec, d_vec, sd_vec, labels=None):
# embedding input vector
q_emb = self.embedding(q_vec)
d_emb = self.embedding(d_vec)
sd_emb = self.embedding(sd_vec)
q_transform = self.q_transformer(q_emb)
d_transform = self.d_transformer(d_emb)
sd_transform = self.sd_transformer(sd_emb)
# Residual Net
q_res = torch.cat((q_transform, q_emb), 2)
d_res = torch.cat((d_transform, d_emb), 2)
sd_res = torch.cat((sd_transform, sd_emb), 2)
# alignment
q_d_similarity = self.similarity(d_res, q_res)
d2q = self.context_to_query(q_d_similarity, d_res)
q2d = self.query_to_context(q_d_similarity, q_res)
q_d_final = self.final_attention(d_res, d2q, q2d)
q_sd_similarity = self.similarity(sd_res, q_res)
sd2q = self.context_to_query(q_sd_similarity, sd_res)
q2sd = self.query_to_context(q_sd_similarity, q_res)
q_sd_final = self.final_attention(sd_res, sd2q, q2sd)
q_d_concat = torch.cat((q_res, q_d_final, d_res),
2) # [batch_size, 128, embedding_size*12]
q_sd_concat = torch.cat((q_res, q_sd_final, sd_res),
2) # [batch_size, 128, embedding_size*12]
q_d_linear = self.q_d_Linear(
q_d_concat) # [batch_size, 128, embedding_size*12]
q_sd_linear = self.q_sd_Linear(
q_sd_concat) # [batch_size, 128, embedding_size*12]
all_concat_input = torch.cat((q_d_linear, q_sd_linear),
2) # [batch_size, 128, embedding_size*24]
all_concat_input = all_concat_input.to(self.device)
result_output = self.result_Linear(
all_concat_input) # [batch_size, 128, embedding_size*24]
result_output = result_output.to(self.device)
result_output = result_output.permute(0, 2, 1)
avg_pool = F.adaptive_avg_pool1d(result_output, 1)
max_pool = F.adaptive_max_pool1d(result_output, 1)
avg_pool = avg_pool.view(q_vec.size(0), -1)
max_pool = max_pool.view(q_vec.size(0), -1)
result = torch.cat((avg_pool, max_pool),
1) # [batch_size, embedding_size*48]
logits = self.classifier(result)
if labels is not None:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits, labels)
return loss, logits
else:
return logits
def forward(self, x):
x = F.adaptive_max_pool1d(x, output_size=7)
x = F.adaptive_max_pool1d(x, output_size=1)
return x
def forward(self, x):
x = self.features(x)
# print(x.size())
b = x.size(0)
x = F.adaptive_max_pool1d(x, 1).view(b, 512)
return x
def forward(self, x):
batch_size = x.size(0)
x = get_graph_feature(
x, k=self.k
) # (batch_size, 3, num_points) -> (batch_size, 3*2, num_points, k)
x = self.conv1(
x
) # (batch_size, 3*2, num_points, k) -> (batch_size, 64, num_points, k)
x1 = x.max(
dim=-1, keepdim=False
)[0] # (batch_size, 64, num_points, k) -> (batch_size, 64, num_points)
x = get_graph_feature(
x1, k=self.k
) # (batch_size, 64, num_points) -> (batch_size, 64*2, num_points, k)
x = self.conv2(
x
) # (batch_size, 64*2, num_points, k) -> (batch_size, 64, num_points, k)
x2 = x.max(
dim=-1, keepdim=False
)[0] # (batch_size, 64, num_points, k) -> (batch_size, 64, num_points)
x = get_graph_feature(
x2, k=self.k
) # (batch_size, 64, num_points) -> (batch_size, 64*2, num_points, k)
x = self.conv3(
x
) # (batch_size, 64*2, num_points, k) -> (batch_size, 128, num_points, k)
x3 = x.max(
dim=-1, keepdim=False
)[0] # (batch_size, 128, num_points, k) -> (batch_size, 128, num_points)
x = get_graph_feature(
x3, k=self.k
) # (batch_size, 128, num_points) -> (batch_size, 128*2, num_points, k)
x = self.conv4(
x
) # (batch_size, 128*2, num_points, k) -> (batch_size, 256, num_points, k)
x4 = x.max(
dim=-1, keepdim=False
)[0] # (batch_size, 256, num_points, k) -> (batch_size, 256, num_points)
x = torch.cat((x1, x2, x3, x4),
dim=1) # (batch_size, 64+64+128+256, num_points)
x = self.conv5(
x
) # (batch_size, 64+64+128+256, num_points) -> (batch_size, emb_dims, num_points)
x1 = F.adaptive_max_pool1d(x, 1).view(
batch_size,
-1) # (batch_size, emb_dims, num_points) -> (batch_size, emb_dims)
x2 = F.adaptive_avg_pool1d(x, 1).view(
batch_size,
-1) # (batch_size, emb_dims, num_points) -> (batch_size, emb_dims)
x = torch.cat((x1, x2), 1) # (batch_size, emb_dims*2)
x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2
) # (batch_size, emb_dims*2) -> (batch_size, 512)
x = self.dp1(x)
x = F.leaky_relu(
self.bn7(self.linear2(x)),
negative_slope=0.2) # (batch_size, 512) -> (batch_size, 256)
x = self.dp2(x)
x = self.linear3(
x) # (batch_size, 256) -> (batch_size, output_channels)
return x
def forward(self, pointcloud: torch.cuda.FloatTensor, cls):
# x: B,3,N
xyz, features = self._break_up_pc(pointcloud)
num_pts = xyz.size(1)
batch_size = xyz.size(0)
# FPS to find different point subsets and their relations
subset1_idx = pointnet2_utils.furthest_point_sample(xyz, num_pts //
4).long() # B,N/2
subset1_xyz = torch.unsqueeze(subset1_idx, -1).repeat(1, 1,
3) # B,N/2,3
subset1_xyz = torch.take(xyz, subset1_xyz) # B,N/2,3
dist, idx1 = pointnet2_utils.three_nn(xyz, subset1_xyz)
dist_recip = 1.0 / (dist + 1e-8)
norm = torch.sum(dist_recip, dim=2, keepdim=True)
weight1 = dist_recip / norm
subset12_idx = pointnet2_utils.furthest_point_sample(
subset1_xyz, num_pts // 16).long() # B,N/4
subset12_xyz = torch.unsqueeze(subset12_idx, -1).repeat(1, 1,
3) # B,N/4,3
subset12_xyz = torch.take(subset1_xyz, subset12_xyz) # B,N/4,3
dist, idx12 = pointnet2_utils.three_nn(subset1_xyz, subset12_xyz)
dist_recip = 1.0 / (dist + 1e-8)
norm = torch.sum(dist_recip, dim=2, keepdim=True)
weight12 = dist_recip / norm
device = torch.device('cuda')
centroid = torch.zeros([batch_size, 1, 3], device=device)
dist, idx0 = pointnet2_utils.three_nn(subset12_xyz, centroid)
dist_recip = 1.0 / (dist + 1e-8)
norm = torch.sum(dist_recip, dim=2, keepdim=True)
weight0 = dist_recip / norm
#######################################
# Error-minimizing module 1:
# Encoding
x = xyz.transpose(2, 1) # x: B,3,N
x1_1 = x
x = get_adptive_dilated_graph_feature(x,
self.conv_op1,
self.conv_op11,
self.conv_op12,
d=5,
k=20)
x = self.conv1(x) # B,64,N,k
x = self.conv14(x) # B,64,N,k
x1_2 = x
# Back-projection
x = self.conv11(x) # B,3,N,1
x = torch.squeeze(x, -1) # B,3,N
x1_3 = x
# Calculating Error
delta_1 = x1_3 - x1_1 # B,3,N
# Output
x = x1_2 # B,64,N,k
x1 = x.max(dim=-1, keepdim=False)[0] # B,64,N
#######################################
#######################################
# Multi-resolution (MR) Branch
# Down-scaling 1
subset1_feat = torch.unsqueeze(subset1_idx, -1).repeat(1, 1,
64) # B,N/2,64
x1_subset1 = torch.take(x1.transpose(1, 2).contiguous(),
subset1_feat).transpose(
1, 2).contiguous() # B,64,N/2
x2_1 = x1_subset1 # B,64,N/2
x = get_graph_feature(x1_subset1, k=self.k // 2)
x = self.conv2(x) # B,64,N/2,k
x = self.conv24(x) # B,128,N/2,k
x2 = x.max(dim=-1, keepdim=False)[0] # B,128,N/2
# Dense-connection
x12 = pointnet2_utils.three_interpolate(x2, idx1, weight1) # B,128,N
x12 = torch.cat((x12, x1), dim=1) # B,192,N
x12 = self.conv23(x12) # B,128,N
# Down-scaling 2
subset12_feat = torch.unsqueeze(subset12_idx,
-1).repeat(1, 1, 128) # B,N/4,128
x2_subset12 = torch.take(
x2.transpose(1, 2).contiguous(),
subset12_feat).transpose(1, 2).contiguous() # B,128,N/4
x3_1 = x2_subset12 # B,128,N/4
x = get_graph_feature(x2_subset12, k=self.k // 4)
x = self.conv3(x) # B,256,N/4,k
x3 = x.max(dim=-1, keepdim=False)[0] # B,256,N/4
# Dense-connection
x23 = pointnet2_utils.three_interpolate(x3, idx12,
weight12) # B,256,N/2
x23 = torch.cat((x23, x2), dim=1) # B,384,N/2
x23 = self.conv34(x23) # B,128,N/2
x123 = pointnet2_utils.three_interpolate(x23, idx1, weight1) # B,128,N
x123 = torch.cat((x123, x12, x1), dim=1) # B,320,N
x123 = self.conv35(x123) # B,128,N
# Down-scaling 3
x_bot = self.conv53(x3)
x_bot = self.conv54(x_bot) # B,1024,N/128
x_bot = F.adaptive_max_pool1d(x_bot, 1) # B,1024,1
# Upsampling 3:
interpolated_feats1 = pointnet2_utils.three_interpolate(
x_bot, idx0, weight0) # B,1024,N/4
interpolated_feats2 = x3 # B,256,N/4
x3_up = torch.cat((interpolated_feats1, interpolated_feats2),
dim=1) # B,1280,N/4
x3_up = self.conv32(x3_up) # B,256,N/4
x3_up = self.conv33(x3_up) # B,256,N/4
# Upsampling 2:
interpolated_feats1 = pointnet2_utils.three_interpolate(
x3_up, idx12, weight12) # B,256,N/2
interpolated_feats2 = x2 # B,128,N/2
interpolated_feats3 = x23 # B,128,N/2
x2_up = torch.cat(
(interpolated_feats1, interpolated_feats3, interpolated_feats2),
dim=1) # B,512,N/2
x2_up = self.conv21(x2_up) # B,256,N/2
x2_up = self.conv22(x2_up) # B,128,N/2
# Upsampling 1:
interpolated_feats1 = pointnet2_utils.three_interpolate(
x2_up, idx1, weight1) # B,128,N
interpolated_feats2 = x1 # B,64,N
interpolated_feats3 = x12 # B,128,N
interpolated_feats4 = x123 # B,128,N
x1_up = torch.cat((interpolated_feats1, interpolated_feats4,
interpolated_feats3, interpolated_feats2),
dim=1) # B,448,N
x1_up = self.conv12(x1_up) # B,512,N
x1_up = self.conv13(x1_up) # B,1024,N
x_mr = x1_up
#############################################################################
#############################################################################
# Full-resolution Branch
# Error-minimizing module 2:
# Encoding
x2_1 = x1 # B,64,N
x = get_adptive_dilated_graph_feature(x1,
self.conv_op2,
self.conv_op21,
self.conv_op22,
d=5,
k=20)
x = self.convfc2(x) # B,64,N,k
x = self.convfc24(x) # B,64,N,k
x2_2 = x
# Back-projection
x = self.convfc21(x) # B,64,N,1
x = torch.squeeze(x, -1) # B,64,N
x2_3 = x
# Calculating Error
delta_2 = x2_3 - x2_1 # B,64,N
# Output
x = x2_2 # B,64,N,k
x2 = x.max(dim=-1, keepdim=False)[0] # B,64,N
#######################################
# Error-minimizing module 3:
# Encoding
x3_1 = x2 # B,64,N
x = get_adptive_dilated_graph_feature(x2,
self.conv_op3,
self.conv_op31,
self.conv_op32,
d=5,
k=20)
x = self.convfc3(x) # B,128,N,k
x3_2 = x
# Back-projection
x = self.convfc31(x) # B,64,N,1
x = torch.squeeze(x, -1) # B,64,N
x3_3 = x
# Calculating Error
delta_3 = x3_3 - x3_1 # B,64,N
# Output
x = x3_2 # B,128,N,k
x3 = x.max(dim=-1, keepdim=False)[0] # B,128,N
#######################################
# Error-minimizing module 4:
# Encoding
x4_1 = x3 # B,128,N
x = get_adptive_dilated_graph_feature(x3,
self.conv_op4,
self.conv_op41,
self.conv_op42,
d=5,
k=20)
x = self.convfc4(x) # B,256,N,k
x4_2 = x
# Back-projection
x = self.convfc41(x) # B,128,N,1
x = torch.squeeze(x, -1) # B,128,N
x4_3 = x
# Calculating Error
delta_4 = x4_3 - x4_1 # B,128,N
# Output
x = x4_2 # B,256,N,k
x4 = x.max(dim=-1, keepdim=False)[0] # B,256,N
x = torch.cat((x1, x2, x3, x4), dim=1) # B,512,N
x_fr = self.conv7(x) # B,1024,N
# Fusing FR and MR outputs
fusion_score = self.fuse(x_mr)
x = x_fr + x_fr * fusion_score
x_all = self.conv9(x) # B,1024,N
# Collecting global feature
one_hot_label = cls.view(-1, 16, 1) # B,16,1
one_hot_label = self.conv5(one_hot_label) # B,64,1
x_max = F.adaptive_max_pool1d(x_all, 1) # B,1024,1
x_global = torch.cat((x_max, one_hot_label), dim=1) # B,1088,1
x_global = x_global.repeat(1, 1, num_pts) # B,1088,N
x = torch.cat((x_all, x_global), dim=1) # B,2112,N
x = self.conv8(x) # B,1024,N
x = self.conv63(x) # B,128,N
x = self.dp(x)
x = self.conv64(x) # B,50,N
return (x.transpose(2,
1).contiguous(), delta_1.transpose(2,
1).contiguous(),
delta_2.transpose(2, 1).contiguous(),
delta_3.transpose(2, 1).contiguous(),
delta_4.transpose(2, 1).contiguous())
def test_adaptive_max_pool1d(self):
inp = torch.randn(1, 16, 28, device='cuda', dtype=self.dtype)
out = F.adaptive_max_pool1d(inp, output_size=5, return_indices=True)
def forward(self, x1_seq, len1, x2_seq, len2, pooling_mode='hstates', device="cpu", output_state_vectors=False, evaluation=False):
if evaluation:
# XXX Set dropouts to zero manually
self.att1_dropout = 0
self.att2_dropout = 0
self.fc1_dropout = 0
self.fc2_dropout = 0
if output_state_vectors:
create_parent_dir(output_state_vectors)
self.h1, self.c1 = self.init_hidden(x1_seq.size(1), device)
x1_embs_not_packed = self.emb(x1_seq)
x1_embs = pack_padded_sequence(x1_embs_not_packed, len1, enforce_sorted=False)
# To avoid the following issue:
# RNN module weights are not part of single contiguous chunk of memory.
# This means they need to be compacted at every call, possibly greatly increasing memory usage.
# To compact weights again call flatten_parameters().
self.rnn_1.flatten_parameters()
if self.main_architecture.lower() in ["lstm"]:
rnn_out_1, (self.h1, self.c1) = self.rnn_1(x1_embs, (self.h1, self.c1))
elif self.main_architecture.lower() in ["gru", "rnn"]:
rnn_out_1, self.h1 = self.rnn_1(x1_embs, self.h1)
rnn_out_1, len1 = pad_packed_sequence(rnn_out_1)
if output_state_vectors:
# the layers can be separated using h_n.view(num_layers, num_directions, batch, hidden_size).
h1_reshape = self.h1.view(self.rnn_n_layers, self.num_directions, rnn_out_1.shape[1], self.rnn_hidden_dim)
output_h_layer = self.rnn_n_layers - 1
if not self.file_id:
self.file_id = len(glob.glob(output_state_vectors + "_fwd_*"))
else:
self.file_id += 1
torch.save(h1_reshape[output_h_layer, 0], f'{output_state_vectors}_fwd_{self.file_id}')
if self.bidirectional:
torch.save(h1_reshape[output_h_layer, 1], f'{output_state_vectors}_bwd_{self.file_id}')
return (h1_reshape[output_h_layer, 0], h1_reshape[output_h_layer, 1])
else:
return (h1_reshape[output_h_layer, 0], False)
if pooling_mode in ['attention']:
attn_weight_flag = False
attn_weight_array = False
for i_nhs in range(np.shape(rnn_out_1)[0]):
attn_weight = F.relu(self.attn_step1(F.dropout(rnn_out_1[i_nhs], self.att1_dropout)))
attn_weight = self.attn_step2(F.dropout(attn_weight, self.att2_dropout))
if not attn_weight_flag:
attn_weight_array = attn_weight
attn_weight_flag = True
else:
attn_weight_array = torch.cat((attn_weight_array, attn_weight), dim=1)
attn_weight_array = F.softmax(attn_weight_array, dim=1)
attn_vect_1 = torch.squeeze(torch.bmm(rnn_out_1.permute(1, 2, 0), torch.unsqueeze(attn_weight_array, 2)))
elif pooling_mode in ['average']:
pool_1 = F.adaptive_avg_pool1d(rnn_out_1.permute(1, 2, 0), 1).view(x1_seq.size(1), -1)
elif pooling_mode in ['max', 'maximum']:
pool_1 = F.adaptive_max_pool1d(rnn_out_1.permute(1, 2, 0), 1).view(x1_seq.size(1), -1)
elif pooling_mode in ['hstates']:
hstates_1_fwd_bwd = self.h1.view(self.rnn_n_layers, self.num_directions, rnn_out_1.shape[1], self.rnn_hidden_dim)
hstates_1 = hstates_1_fwd_bwd[self.rnn_n_layers - 1, 0]
if self.bidirectional:
hstates_1 = torch.cat((hstates_1, hstates_1_fwd_bwd[self.rnn_n_layers - 1, 1]), dim=1)
elif pooling_mode in ['hstates_layers', 'hstates_layers_simple', 'hstates_subtract',
'hstates_l2_distance', 'hstates_cosine']:
hstates_1_fwd_bwd = self.h1.view(self.rnn_n_layers, self.num_directions, rnn_out_1.shape[1], self.rnn_hidden_dim)
hstates_1 = hstates_1_fwd_bwd[0, 0]
for rlayer in range(1, self.rnn_n_layers):
hstates_1 = torch.cat((hstates_1, hstates_1_fwd_bwd[rlayer, 0]), dim=1)
if self.bidirectional:
hstates_1_bwd = hstates_1_fwd_bwd[0, 1]
for rlayer in range(1, self.rnn_n_layers):
hstates_1_bwd = torch.cat((hstates_1_bwd, hstates_1_fwd_bwd[rlayer, 1]), dim=1)
hstates_1 = torch.cat((hstates_1, hstates_1_bwd), dim=1)
self.h2, self.c2 = self.init_hidden(x2_seq.size(1), device)
x2_embs_not_packed = self.emb(x2_seq)
x2_embs = pack_padded_sequence(x2_embs_not_packed, len2, enforce_sorted=False)
# Share parameters between two GRUs
# Previously, we had gru_out_2, self.h2 = self.gru_2(x2_embs, self.h2)
if self.main_architecture.lower() in ["lstm"]:
rnn_out_2, (self.h2, self.c2) = self.rnn_1(x2_embs, (self.h2, self.c2))
elif self.main_architecture.lower() in ["gru", "rnn"]:
rnn_out_2, self.h2 = self.rnn_1(x2_embs, self.h2)
rnn_out_2, len2 = pad_packed_sequence(rnn_out_2)
if pooling_mode in ['attention']:
attn_weight_flag = False
attn_weight_array = False
for i_nhs in range(np.shape(rnn_out_2)[0]):
attn_weight = F.relu(self.attn_step1(F.dropout(rnn_out_2[i_nhs], self.att1_dropout)))
attn_weight = self.attn_step2(F.dropout(attn_weight, self.att2_dropout))
if not attn_weight_flag:
attn_weight_array = attn_weight
attn_weight_flag = True
else:
attn_weight_array = torch.cat((attn_weight_array, attn_weight), dim=1)
attn_weight_array = F.softmax(attn_weight_array, dim=1)
attn_vect_2 = torch.squeeze(torch.bmm(rnn_out_2.permute(1, 2, 0), torch.unsqueeze(attn_weight_array, 2)))
elif pooling_mode in ['average']:
pool_2 = F.adaptive_avg_pool1d(rnn_out_2.permute(1, 2, 0), 1).view(x2_seq.size(1), -1)
elif pooling_mode in ['max', 'maximum']:
pool_2 = F.adaptive_max_pool1d(rnn_out_2.permute(1, 2, 0), 1).view(x2_seq.size(1), -1)
elif pooling_mode in ['hstates']:
hstates_2_fwd_bwd = self.h2.view(self.rnn_n_layers, self.num_directions, rnn_out_2.shape[1], self.rnn_hidden_dim)
hstates_2 = hstates_2_fwd_bwd[self.rnn_n_layers - 1, 0]
if self.bidirectional:
hstates_2 = torch.cat((hstates_2, hstates_2_fwd_bwd[self.rnn_n_layers - 1, 1]), dim=1)
elif pooling_mode in ['hstates_layers', 'hstates_layers_simple', 'hstates_subtract',
'hstates_l2_distance', 'hstates_cosine']:
hstates_2_fwd_bwd = self.h2.view(self.rnn_n_layers, self.num_directions, rnn_out_2.shape[1], self.rnn_hidden_dim)
hstates_2 = hstates_2_fwd_bwd[0, 0]
for rlayer in range(1, self.rnn_n_layers):
hstates_2 = torch.cat((hstates_2, hstates_2_fwd_bwd[rlayer, 0]), dim=1)
if self.bidirectional:
hstates_2_bwd = hstates_2_fwd_bwd[0, 1]
for rlayer in range(1, self.rnn_n_layers):
hstates_2_bwd = torch.cat((hstates_2_bwd, hstates_2_fwd_bwd[rlayer, 1]), dim=1)
hstates_2 = torch.cat((hstates_2, hstates_2_bwd), dim=1)
# Combine outputs from GRU1 and GRU2
if pooling_mode in ['attention']:
attn_vec_cat = torch.cat((attn_vect_1, attn_vect_2), dim=1)
attn_vec_mul = attn_vect_1 * attn_vect_2
attn_vec_dif = attn_vect_1 - attn_vect_2
output_combined = torch.cat((attn_vec_cat,
attn_vec_mul,
attn_vec_dif), dim=1)
elif pooling_mode in ['average', 'max', 'maximum']:
pool_rnn_cat = torch.cat((pool_1, pool_2), dim=1)
pool_rnn_mul = pool_1 * pool_2
pool_rnn_dif = pool_1 - pool_2
output_combined = torch.cat((pool_rnn_cat,
pool_rnn_mul,
pool_rnn_dif), dim=1)
elif pooling_mode in ['hstates', 'hstates_layers']:
hstates_rnn_cat = torch.cat((hstates_1, hstates_2), dim=1)
hstates_rnn_mul = hstates_1 * hstates_2
hstates_rnn_dif = hstates_1 - hstates_2
output_combined = torch.cat((hstates_rnn_cat,
hstates_rnn_mul,
hstates_rnn_dif), dim=1)
elif pooling_mode in ['hstates_layers_simple']:
output_combined = torch.cat((hstates_1, hstates_2), dim=1)
elif pooling_mode in ['hstates_subtract']:
output_combined = 1 - torch.abs(hstates_1 - hstates_2)
elif pooling_mode in ['hstates_l2_distance']:
output_combined = 1 - torch.abs(hstates_1 - hstates_2)**2
elif pooling_mode in ['hstates_cosine']:
hstates_cosine_sim = nn.CosineSimilarity(dim=1, eps=1e-10)(hstates_1, hstates_2)
# in this case, return the cosine similarity as predictions
#return torch.log(torch.stack([1- hstates_cosine_sim, hstates_cosine_sim])).T
return torch.stack([1-hstates_cosine_sim, hstates_cosine_sim]).T
y_out = F.relu(self.fc1(F.dropout(output_combined, self.fc1_dropout)))
y_out = self.fc2(F.dropout(y_out, self.fc2_dropout))
return y_out
def forward(self, x, phase='train'):
out = self.conv(x)
# print out.size()
out = self.bn(out)
# print out.size()
out = self.relu(out)
# print out.size()
out1 = self.layer1(out) # 64
# print out1.size()
out2 = self.layer2(out1) # 32
# print out2.size()
out3 = self.layer3(out2) # 16
# print out3.size()
out4 = self.layer4(out3) # 8
# print out4.size()
out5 = self.layer5(out4) # 4
# print out5.size()
# out = F.adaptive_max_pool2d(out5, output_size=(1,1)).view(out.size(0), -1) # 128
# out = out.view(out.size(0), -1)
if phase == 'seg':
out = F.adaptive_max_pool2d(out5,
output_size=(1,
1)).view(out.size(0),
-1) # 128
out = self.fc(out)
out = out.view(out.size(0), -1)
else:
out = F.max_pool2d(out5, 2)
out_size = out.size()
# out = out.view(out_size[0],out_size[1],out_size[3]).transpose(1,2).contiguous().view(-1, out_size[1])
out = out.view(out_size[0],
out_size[1], out_size[2] * out_size[3]).transpose(
1, 2).contiguous().view(-1, out_size[1])
out = self.fc(out)
out = out.view(out_size[0], out_size[2] * out_size[3],
-1).transpose(1, 2).contiguous()
out = F.adaptive_max_pool1d(out,
output_size=(1)).view(out_size[0], -1)
# print out.size()
if phase not in ['seg', 'pretrain', 'pretrain2']:
return out
# detect
cat1 = torch.cat([self.convt1(out5), out4], 1)
# print cat1.size()
dec1 = self.dec1(cat1)
# print dec1.size()
# print out3.size()
cat2 = torch.cat([self.convt2(dec1), out3], 1)
# print cat2.size()
dec2 = self.dec2(cat2)
cat3 = torch.cat([self.convt3(dec2), out2], 1)
dec3 = self.dec3(cat3)
cat4 = torch.cat([self.convt4(dec3), out1], 1)
seg = self.dec4(cat4)
seg = seg.view((seg.size(0), seg.size(2), seg.size(3)))
seg = self.sigmoid(seg)
bbox = self.bbox(cat2)
# dec2 = self.output(dec2)
# print dec2.size()
size = bbox.size()
bbox = bbox.view((size[0], size[1], -1)).transpose(1, 2).contiguous()
bbox = bbox.view((size[0], size[2], size[3], -1, 4))
return out, bbox, seg
def forward(self, sent_m, mention_m, relations_m, relations_words_m,
entities_m, candidates_m, candidates_labels_m, features_m):
choices = candidates_labels_m.size(
1) # number of possible candidates per one mention
real_choices_num = torch.sum((candidates_m > 0).float(),
dim=1).unsqueeze(1)
sent_emb = self._words2vector(sent_m).unsqueeze(1)
mention_emb = self._chars2vector(mention_m).unsqueeze(1)
sent_emb_expanded = sent_emb.expand(sent_emb.size(0), choices,
sent_emb.size(2)).contiguous()
mention_emb_expanded = mention_emb.expand(
mention_emb.size(0), choices, mention_emb.size(2)).contiguous()
relations_words_m = relations_words_m.view(
relations_words_m.size(0) * relations_words_m.size(1) *
relations_words_m.size(2), -1)
relations_m = relations_m.view(
relations_m.size(0) * relations_m.size(1), -1)
entities_m = entities_m.view(
entities_m.size(0) * entities_m.size(1), -1)
candidates_labels_m = candidates_labels_m.view(
candidates_labels_m.size(0) * candidates_labels_m.size(1), -1, 2)
candidate_vectors = self.compute_candidate_vectors(
relations_m, relations_words_m, entities_m, candidates_m,
candidates_labels_m)
candidate_vectors = candidate_vectors.view(-1, choices,
candidate_vectors.size(-1))
features_m = features_m.float()
concatenated_embed = torch.cat(
(sent_emb_expanded, mention_emb_expanded, candidate_vectors,
features_m),
dim=-1).contiguous()
concatenated_embed = concatenated_embed.view(
-1, concatenated_embed.size(-1))
sem_vector = self.sem_layers(concatenated_embed)
sem_vector = sem_vector.view(-1, choices, sem_vector.size(-1))
sem_vector_pooled_over_choices = sem_vector.transpose(-2, -1)
sem_vector_pooled_over_choices = self._pool(
sem_vector_pooled_over_choices)
sem_vector_pooled_over_choices = sem_vector_pooled_over_choices.transpose(
-2, -1)
sem_vector = torch.cat(
(sem_vector, sem_vector_pooled_over_choices.expand_as(sem_vector)),
dim=-1)
sem_vector = sem_vector.view(-1, sem_vector.size(-1))
candidate_scores = self.score_weights(sem_vector).squeeze(dim=-1)
candidate_scores = candidate_scores.view(-1, choices)
negative_vector = torch.cat(
(sent_emb.squeeze(dim=1), mention_emb.squeeze(dim=1)), dim=1)
negative_vector = self.negative_layers(negative_vector)
real_choices_num = self._nonlinearity(real_choices_num)
choices_pooled_for_negative = sem_vector_pooled_over_choices.squeeze(
dim=1)
negative_vector = torch.cat(
(negative_vector, choices_pooled_for_negative,
F.adaptive_max_pool1d(candidate_scores.unsqueeze(1),
1).squeeze(dim=-1), real_choices_num),
dim=-1)
negative_score = self.negative_weight(negative_vector)
candidate_scores = self._nonlinearity(candidate_scores)
return F.sigmoid(negative_score.squeeze(dim=-1)), candidate_scores
def forward(self, x):
batch_size = x.size(0)
# shape of x : batch, feature, npoints, neighbors
# convolution(shared mlp to xi, xj-xi & max)
# =>
# transformer(shared wq, wk, wv to xi)
if self.ape:
ptcld = x
else:
ptcld = None
#print("1")
x, abs_x = get_neighbors(x,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn1) # b, 3, 1024, 20
x1 = self.conv1(x, abs_x) # b, 64, 1024
x1 = self.relu(self.bn1(x1)).squeeze(3)
#print("2")
x, abs_x = get_neighbors(x1,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn2,
ape=self.pos_nn2,
ptcld=ptcld) # b, 64, 1024, 20
x2 = self.conv2(x, abs_x) # b, 64, 1024
x2 = self.relu(self.bn2(x2)).squeeze(3)
#print("3")
x, abs_x = get_neighbors(x2,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn3,
ape=self.pos_nn3,
ptcld=ptcld) # b, 128, 1024, 20
x3 = self.conv3(x, abs_x) # b, 128, 1024
x3 = self.relu(self.bn3(x3)).squeeze(3)
#print("4")
x, abs_x = get_neighbors(x3,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn4,
ape=self.pos_nn4,
ptcld=ptcld) # b, 256, 1024, 20
x4 = self.conv4(x, abs_x) # b, 256, 1024, 20
x4 = self.relu(self.bn4(x4)).squeeze(3)
x = self.conv5(x4)
x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1)
x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2)
x = self.dp1(x)
x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2)
x = self.dp2(x)
x = self.linear3(x)
return x
def aggregate(self, inputs): aggregated = func.adaptive_max_pool1d(inputs, 1) return aggregated
def pool(self, input):
return F.adaptive_max_pool1d(input, 1)
def forward(self, x):
batch_size = x.size(0)
num_points_1 = x.size(2)
# num_points_2 = int(num_points_1/2)
# num_points_3 = int(num_points_1/4)
num_points_2 = int(num_points_1 * 0.75)
num_points_3 = int(num_points_2 * 0.75)
num_points_4 = int(num_points_3 * 0.75)
num_points_5 = int(num_points_4 * 0.75)
########################BLOCK1##############################
N1_points = x.permute(0, 2, 1).contiguous()
x1 = self.GSCM(N1_points, None, self.k, self.conv1, isFirstLayer=True)
########################BLOCK2##############################
fps_id_2 = pointnet2_utils.furthest_point_sample(
N1_points, num_points_2)
N2_points = (pointnet2_utils.gather_operation(
N1_points.transpose(1, 2).contiguous(),
fps_id_2).transpose(1, 2).contiguous())
x1_downSample = (pointnet2_utils.gather_operation(x1, fps_id_2))
x2 = self.GSCM(N2_points, x1_downSample, self.k, self.conv2)
########################BLOCK3##############################
fps_id_3 = pointnet2_utils.furthest_point_sample(
N2_points, num_points_3)
N3_points = (pointnet2_utils.gather_operation(
N2_points.transpose(1, 2).contiguous(),
fps_id_3).transpose(1, 2).contiguous())
x2_downSample = (pointnet2_utils.gather_operation(x2, fps_id_3))
x1_downSample = (pointnet2_utils.gather_operation(
x1_downSample, fps_id_3))
x3 = self.GSCM(N3_points, x2_downSample, self.k, self.conv3)
########################BLOCK4##############################
fps_id_4 = pointnet2_utils.furthest_point_sample(
N3_points, num_points_4)
N4_points = (pointnet2_utils.gather_operation(
N3_points.transpose(1, 2).contiguous(),
fps_id_4).transpose(1, 2).contiguous())
x3_downSample = (pointnet2_utils.gather_operation(x3, fps_id_4))
x2_downSample = (pointnet2_utils.gather_operation(
x2_downSample, fps_id_4))
x1_downSample = (pointnet2_utils.gather_operation(
x1_downSample, fps_id_4))
x4 = self.GSCM(N4_points, x3_downSample, self.k, self.myconv4)
########################BLOCK5##############################
fps_id_5 = pointnet2_utils.furthest_point_sample(
N4_points, num_points_5)
N5_points = (pointnet2_utils.gather_operation(
N4_points.transpose(1, 2).contiguous(),
fps_id_5).transpose(1, 2).contiguous())
x4_downSample = (pointnet2_utils.gather_operation(x4, fps_id_5))
x3_downSample = (pointnet2_utils.gather_operation(
x3_downSample, fps_id_5))
x2_downSample = (pointnet2_utils.gather_operation(
x2_downSample, fps_id_5))
x1_downSample = (pointnet2_utils.gather_operation(
x1_downSample, fps_id_5))
x5 = self.GSCM(N5_points, x4_downSample, self.k, self.myconv5)
x = torch.cat(
(x1_downSample, x2_downSample, x3_downSample, x4_downSample, x5),
dim=1)
x = self.conv5(x)
x1 = F.adaptive_max_pool1d(x, 1).view(batch_size, -1)
x2 = F.adaptive_avg_pool1d(x, 1).view(batch_size, -1)
x = torch.cat((x1, x2), 1)
x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2)
x = self.dp1(x)
x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2)
x = self.dp2(x)
x = self.linear3(x)
return x
def forward(self, x):
batch_size = x.size(0)
# shape of x : batch, feature, npoints, neighbors
# convolution(shared mlp to xi, xj-xi & max)
# =>
# transformer(shared wq, wk, wv to xi)
if self.ape:
ptcld = x
else:
ptcld = None
x, abs_x, deg1, idx1 = get_neighbors(x,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn1,
layer=1) # b, 64, 1024, 20
#print("1")
idx = idx1
x1, k1, v1 = self.conv1(x, abs_x, deg1, idx) # b, 64, 1024
x1 = self.act1(self.bn1(x1)).squeeze(3)
k = k1
v = v1
#print("2")
x, abs_x, deg2, idx2 = get_neighbors(x1,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn2,
layer=2) # b, 64, 1024, 20
x2, k2, v2 = self.conv2(x, abs_x, deg2, idx2, k=k, v=v) # b, 64, 1024
x2 = self.act2(self.bn2(x2)).squeeze(3)
k = torch.cat((k, k2), dim=3) # b, 64, 1024, 2
v = torch.cat((v, v2), dim=3) # b, 64, 1024, 2
#print("3")
x, abs_x, deg3, idx3 = get_neighbors(x2,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn3,
layer=3) # b, 64, 1024, 20
x3, k3, v3 = self.conv3(x, abs_x, deg3, idx3, k=k, v=v) # b, 128, 1024
x3 = self.act3(self.bn3(x3)).squeeze(3)
k = torch.cat((k, k3), dim=3) # b, 64, 1024, 3
v = torch.cat((v, v3), dim=3) # b, 64, 1024, 3
#print("4")
x, abs_x, deg4, idx4 = get_neighbors(x3,
k=self.k,
deg=self.deg,
delta=self.delta,
neighbors=self.neighbors,
bn=self.deg_bn4,
layer=4) # b, 64, 1024, 20
x4, k4, v4 = self.conv4(x, abs_x, deg4, idx4, k=k,
v=v) # b, 256, 1024, 20
x4 = self.act4(self.bn4(x4)).squeeze(3)
x = self.conv5(x4)
x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1)
x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2)
x = self.dp1(x)
x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2)
x = self.dp2(x)
x = self.linear3(x)
return x
def forward(self, x):
b = x.size(0) # batch size
# d = x.size(1) # dimensionality of input
n = x.size(2) # number of points
# EdgeConv ------------------------------------
x = get_graph_feature(x,
rsize=self.rsize,
diff_features_only=self.diff_features_only
) # (b, d, n) -> (b, 2*d, n, F)
x = self.dpR1(x) # (b, 2*d, n) -> (b, 2*d, n, F)
x = self.convR1C1(x) # (b, 2*d, n, F) -> (b, 64, n, F)
x = self.convR1C2(x) # (b, 64, n, F) -> (b, 64, n, F)
r1 = x.max(dim=-1, keepdim=False)[0] # (b, 64, n, F) -> (b, 64, n)
# ---------------------------------------------
# EdgeConv ------------------------------------
x = get_graph_feature(r1,
rsize=self.rsize,
diff_features_only=self.diff_features_only
) # (b, 64, n) -> (b, 64*2, n, F)
x = self.dpR2(x) # (b, 2*d, n) -> (b, 2*d, n, F)
x = self.convR2C1(x) # (b, 64*2, n, F) -> (b, 64, n, F)
x = self.convR2C2(x) # (b, 64, n, F) -> (b, 64, n, F)
r2 = x.max(dim=-1, keepdim=False)[0] # (b, 64, n, F) -> (b, 64, n)
# ---------------------------------------------
x = torch.cat((r1, r2), 1) # (b, 128, n)
x = self.convC1(x) # (b, 128, n) -> (b, 256, n)
# GlobConv ------------------------------------
# GlobExt ----------
x_glob = self.convG11(x) # (b, 256, n) -> (b, 32, n)
x_max = F.adaptive_max_pool1d(x_glob, 1) # (b, 32, n) -> (b, 32, 1)
x_max = x_max.view(b, -1) # (b, 32, 1) -> (b, 32)
x_avg = F.adaptive_avg_pool1d(x_glob, 1) # (b, 32, n) -> (b, 32, 1)
x_avg = x_avg.view(b, -1) # (b, 32, 1) -> (b, 32)
x_glob = torch.cat((x_max, x_avg), 1) # (b, 64)
x_glob = self.convG12(x_glob).unsqueeze(-1) # (b, 64) -> (b, 32, 1)
# ------------------
x = self.convC2(x) # (b, 256, n) -> (b, 96, n)
x = torch.cat((x, x_glob.repeat(1, 1, n)),
1) # (b, 96, n) -> (b, 128, n)
x = self.convC3(x) # (b, 128, n) -> (b, 64, n)
# ---------------------------------------------
x = self.dp(x) # (b, 64, n) -> (b, 64, n))
x = self.convOut(x) # (b, 64, n) -> (b, out_channels, n)
return x
def forward(self, x):
return torch.cat(
(F.adaptive_avg_pool1d(x, 1), F.adaptive_max_pool1d(x, 1)), dim=1)
def forward(self, x):
x = F.relu(self.bn1(self.conv1(x)))
x = F.relu(self.bn2(self.conv2(x)))
x = F.adaptive_max_pool1d(x, 1).squeeze()
return x
def adaptive_catavgmax_pool1d(x, output_size=1):
x_avg = F.adaptive_avg_pool1d(x, output_size)
x_max = F.adaptive_max_pool1d(x, output_size)
return torch.cat((x_avg, x_max), 1)
lstm = nn.LSTM(8, 10, 1, bidirectional=True)
batch_size = input3.size(0)
print(batch_size)
h = Variable(torch.zeros((2, 2, 10)))
c = Variable(torch.zeros((2, 2, 10)))
lstm_out, (h, c) = lstm(output3, (h, c))
print(
lstm_out,
lstm_out.shape) #lstm out => (sequence len, batch size, hidden_dim*2)
# (batch_size, hidden_dim*2, sequence_len)
avg_pool = F.adaptive_avg_pool1d(lstm_out.permute(1, 2, 0),
1).view(batch_size, -1)
max_pool = F.adaptive_max_pool1d(lstm_out.permute(1, 2, 0),
1).view(batch_size, -1)
sentence_output = torch.cat(
[c[-1], avg_pool, max_pool],
dim=1) # (batch_size, 3*num_directions*n_hidden)
print(sentence_output.shape)
all_tog = torch.cat([output, sentence_output], dim=1)
linear = nn.Linear(61, 1)
out = linear(all_tog)
print(out)
