def forward(self, x): x, indices = F.adaptive_max_pool1d(x, output_size=7, return_indices=True) x = F.adaptive_max_pool1d(x, output_size=1) return x, indices
def forward(self, input: Tensor) -> Tensor: input = self.quant_handle(input) return F.adaptive_max_pool1d(input, self.output_size, self.return_indices)
def adaptive_avgmax_pool1d(x, output_size=1): x_avg = F.adaptive_avg_pool1d(x, output_size) x_max = F.adaptive_max_pool1d(x, output_size) return 0.5 * (x_avg + x_max)
def _forward_features(self, x): for l in self.conv: x = F.relu(l(x)) x = F.adaptive_max_pool1d(x, output_size=1) return x
def forward(self, x): batch_size = x.size(0) x = x.repeat(1, 1, 8) #print(x.shape) # shape of x : batch, feature, npoints, neighbors # => # transformer(shared wq, wk, wv to xi) i = 1 points = x tic = time.time() tic_init = tic i, tic = memchk(i, string="points", tic=tic) x, abs_x, idx1 = get_neighbors(x, k=self.k) # b, 64, 1024, 20 i, tic = memchk(i, string="kNN1", tic=tic) x1 = self.conv1(x, abs_x, idx1, points) # b, 64, 1024 i, tic = memchk(i, string="conv1", tic=tic) x1 = self.act1(self.bn1(x1)).squeeze(3) i, tic = memchk(i, string="bnrelu1", tic=tic) x, abs_x, idx2 = get_neighbors(x1, k=self.k) # b, 64, 1024, 20 i, tic = memchk(i, string="kNN2", tic=tic) x2 = self.conv2(x, abs_x, idx2, points) # b, 64, 1024 i, tic = memchk(i, string="conv2", tic=tic) x2 = self.act2(self.bn2(x2)).squeeze(3) i, tic = memchk(i, string="bnrelu2", tic=tic) x, abs_x, idx3 = get_neighbors(x2, k=self.k) # b, 64, 1024, 20 i, tic = memchk(i, string="kNN3", tic=tic) x3 = self.conv3(x, abs_x, idx3, points) # b, 128, 1024 i, tic = memchk(i, string="conv3", tic=tic) x3 = self.act3(self.bn3(x3)).squeeze(3) i, tic = memchk(i, string="bnrelu3", tic=tic) x, abs_x, idx4 = get_neighbors(x3, k=self.k) # b, 64, 1024, 20 i, tic = memchk(i, string="kNN4", tic=tic) x4 = self.conv4(x, abs_x, idx4, points) # b, 256, 1024, 20 i, tic = memchk(i, string="conv4", tic=tic) x4 = self.act4(self.bn4(x4)).squeeze(3) i, tic = memchk(i, string="bnrelu4", tic=tic) x = self.conv5(x4) i, tic = memchk(i, string="conv5", tic=tic) # 8 x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1) x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2) x = self.dp1(x) i, tic = memchk(i, string="dp1", tic=tic) # 8 x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2) x = self.dp2(x) i, tic = memchk(i, string="dp2", tic=tic) # 8 x = self.linear3(x) i, tic_final = memchk(i, string="fc", tic=tic) # 8 print("Total elapsed time : {} ms".format( round((tic_final - tic_init) * 1000, 3))) return x
def forward( self, x_diag: torch.LongTensor, x_lengths: torch.LongTensor, x_timedels: torch.FloatTensor = None, x_age: torch.FloatTensor = None, x_gender: torch.LongTensor = None, x_time_delta_mean: torch.FloatTensor = None, all_input_zeros: bool = False, ) -> torch.Tensor: batch_size, max_seq_length, _ = x_diag.shape if all_input_zeros: x_diag = torch.zeros_like(x_diag) #### Embeddings x_embed = self.diag_embed(x_diag) ## rNN Layers # This does early fusion with feature replication x_gender = (self.gender_embed(x_gender).unsqueeze(1).expand( batch_size, max_seq_length, -1)) x_age = x_age.unsqueeze(1).expand(batch_size, max_seq_length, -1) x_time_delta_mean = x_time_delta_mean.unsqueeze(1).expand( batch_size, max_seq_length, -1) if self.use_adder: x_embed_sum = self.embed_sum(x_embed) else: x_embed_sum = self.embed_conv(x_embed.permute(0, 2, 1, 3)).squeeze(1) # we replicate the scalar features across every time-step x_embed_sum = torch.cat( [ x_embed_sum, x_gender, x_age, x_time_delta_mean, x_timedels.unsqueeze(2) ], dim=2, ) rnn_outs, _ = self.rnn(x_embed_sum) rnn_outs = rnn_outs.contiguous().view(batch_size, max_seq_length, self.rnn_hidden_size) attn_out, _ = self.attn(rnn_outs, rnn_outs, rnn_outs) rnn_attn_max = F.adaptive_max_pool1d(attn_out.permute(0, 2, 1), 1).squeeze() rnn_out = gather_last_relevant_hidden(rnn_outs, x_lengths) if self.use_mh_attn: rnn_out = (1 - F.sigmoid(self.attn_alpha) ) * rnn_out + rnn_attn_max * F.sigmoid(self.attn_alpha) z = self.fc_w_feats(rnn_out) if self.do_bn: z = self.ln(z) z = self.dropout(self.activation(z)) y_pred = self.out(z) y_out = F.relu(y_pred.view(-1)) return y_out
def forward(self, x, norm_plt, cls_label, gt=None): B, C, N = x.size() idx, _ = knn( x, k=self.k ) # different with DGCNN, the knn search is only in 3D space xyz = get_scorenet_input(x, k=self.k, idx=idx) # ScoreNet input ################# # use MLP at the 1st layer, same with DGCNN x = get_graph_feature(x, k=self.k, idx=idx) x = x.permute(0, 3, 1, 2) # b,2cin,n,k x = F.relu(self.conv1(x)) x1 = x.max(dim=-1, keepdim=False)[0] ################# # replace the last 4 DGCNN-EdgeConv with PAConv: """CUDA implementation of PAConv: (presented in the supplementary material of the paper)""" """feature transformation:""" x2, center2 = feat_trans_dgcnn(point_input=x1, kernel=self.matrice2, m=self.m2) score2 = self.scorenet2(xyz, calc_scores=self.calc_scores, bias=0) """assemble with scores:""" x = assemble_dgcnn(score=score2, point_input=x2, center_input=center2, knn_idx=idx, aggregate='sum') x2 = F.relu(self.bn2(x)) x3, center3 = feat_trans_dgcnn(point_input=x2, kernel=self.matrice3, m=self.m3) score3 = self.scorenet3(xyz, calc_scores=self.calc_scores, bias=0) x = assemble_dgcnn(score=score3, point_input=x3, center_input=center3, knn_idx=idx, aggregate='sum') x3 = F.relu(self.bn3(x)) x4, center4 = feat_trans_dgcnn(point_input=x3, kernel=self.matrice4, m=self.m4) score4 = self.scorenet4(xyz, calc_scores=self.calc_scores, bias=0) x = assemble_dgcnn(score=score4, point_input=x4, center_input=center4, knn_idx=idx, aggregate='sum') x4 = F.relu(self.bn4(x)) x5, center5 = feat_trans_dgcnn(point_input=x4, kernel=self.matrice5, m=self.m5) score5 = self.scorenet5(xyz, calc_scores=self.calc_scores, bias=0) x = assemble_dgcnn(score=score5, point_input=x5, center_input=center5, knn_idx=idx, aggregate='sum') x5 = F.relu(self.bn5(x)) ############### xx = torch.cat((x1, x2, x3, x4, x5), dim=1) xc = F.relu(self.convt(xx)) xc = F.adaptive_max_pool1d(xc, 1).view(B, -1) cls_label = cls_label.view(B, 16, 1) cls_label = F.relu(self.convc(cls_label)) cls = torch.cat((xc.view(B, 1024, 1), cls_label), dim=1) cls = cls.repeat(1, 1, N) # B,1088,N x = torch.cat((xx, cls), dim=1) # 1088+64*3 x = F.relu(self.conv6(x)) x = self.dp1(x) x = F.relu(self.conv7(x)) x = self.dp2(x) x = F.relu(self.conv8(x)) x = self.conv9(x) x = F.log_softmax(x, dim=1) x = x.permute(0, 2, 1) # b,n,50 if gt is not None: return x, F.nll_loss(x.contiguous().view(-1, self.num_part), gt.view(-1, 1)[:, 0]) else: return x
def forward(self, x, label=None, criterion=None): B, C, N = x.size() idx, _ = knn( x, k=self.k ) # different with DGCNN, the knn search is only in 3D space xyz = get_scorenet_input( x, idx=idx, k=self.k ) # ScoreNet input: 3D coord difference concat with coord: b,6,n,k ################## # replace all the DGCNN-EdgeConv with PAConv: """CUDA implementation of PAConv: (presented in the supplementary material of the paper)""" """feature transformation:""" point1, center1 = feat_trans_dgcnn(point_input=x, kernel=self.matrice1, m=self.m1) # b,n,m1,o1 score1 = self.scorenet1(xyz, calc_scores=self.calc_scores, bias=0.5) """assemble with scores:""" point1 = assemble_dgcnn(score=score1, point_input=point1, center_input=center1, knn_idx=idx, aggregate='sum') # b,o1,n point1 = F.relu(self.bn1(point1)) point2, center2 = feat_trans_dgcnn(point_input=point1, kernel=self.matrice2, m=self.m2) score2 = self.scorenet2(xyz, calc_scores=self.calc_scores, bias=0.5) point2 = assemble_dgcnn(score=score2, point_input=point2, center_input=center2, knn_idx=idx, aggregate='sum') point2 = F.relu(self.bn2(point2)) point3, center3 = feat_trans_dgcnn(point_input=point2, kernel=self.matrice3, m=self.m3) score3 = self.scorenet3(xyz, calc_scores=self.calc_scores, bias=0.5) point3 = assemble_dgcnn(score=score3, point_input=point3, center_input=center3, knn_idx=idx, aggregate='sum') point3 = F.relu(self.bn3(point3)) point4, center4 = feat_trans_dgcnn(point_input=point3, kernel=self.matrice4, m=self.m4) score4 = self.scorenet4(xyz, calc_scores=self.calc_scores, bias=0.5) point4 = assemble_dgcnn(score=score4, point_input=point4, center_input=center4, knn_idx=idx, aggregate='sum') point4 = F.relu(self.bn4(point4)) ################## point = torch.cat((point1, point2, point3, point4), dim=1) point = F.relu(self.conv5(point)) point11 = F.adaptive_max_pool1d(point, 1).view(B, -1) point22 = F.adaptive_avg_pool1d(point, 1).view(B, -1) point = torch.cat((point11, point22), 1) point = F.relu(self.bn11(self.linear1(point))) point = self.dp1(point) point = F.relu(self.bn22(self.linear2(point))) point = self.dp2(point) point = self.linear3(point) if criterion is not None: return point, criterion(point, label) # return output and loss else: return point
def forward(self, x, vars=None, bn_training=True): if vars is None: vars = self.vars idx = 0 bn_idx = 0 for name, param in self.config: if name is 'conv1d': w, b = vars[idx], vars[idx + 1] # remember to keep synchrozied of forward_encoder and forward_decoder! x = F.conv1d(x, w, b, stride=param[3], padding=param[4]) idx += 2 if param[5] == 1: x1 = x elif param[5] == 2: x2 = x elif param[5] == 3: x3 = x elif name is 'concat': x = torch.cat([x1, x2, x3], 1) elif name is 'linear': w, b = vars[idx], vars[idx + 1] x = F.linear(x, w, b) idx += 2 elif name is 'bn': w, b = vars[idx], vars[idx + 1] running_mean, running_var = self.vars_bn[bn_idx], self.vars_bn[ bn_idx + 1] x = F.batch_norm(x, running_mean, running_var, weight=w, bias=b, training=bn_training) idx += 2 bn_idx += 2 elif name is 'flatten': # print(x.shape) x = x.view(x.size(0), -1) elif name is 'relu': x = F.relu(x, inplace=param[0]) elif name is 'max_pool1d': x = F.max_pool1d(x, param[0]) if param[1] == 1: x1 = x elif name is 'SE': x_se = F.adaptive_max_pool1d(x, param[0]) x_se = x_se.view(x_se.size(0), -1) w1, b1 = vars[idx], vars[idx + 1] x_se = F.linear(x_se, w1, b1) idx += 2 x_se = F.relu(x_se, inplace=param[3]) w2, b2 = vars[idx], vars[idx + 1] x_se = F.linear(x_se, w2, b2) idx += 2 x_se = F.sigmoid(x_se).unsqueeze(2) x_se = x_se.expand_as(x) x = x * x_se else: raise NotImplementedError assert idx == len(vars) assert bn_idx == len(self.vars_bn) return x
def forward(self, bins): lst_encodings = [] bio_targets = [] ec_targets = [] event_duration_targets = [] independent_events_targets = [] ec_independent_targets = [] ec_independent_event_ids = [] for bin in bins: bin_tweets = bin[0] bin_tweets = bin_tweets.to('cuda') bin_tweets = bin_tweets.squeeze(0) bio_targets.extend(bin[1].data.tolist()) ec_targets.extend(bin[2]) event_duration_targets.extend( bin[3]) #(0,1) with duration 0000111111111111110000 independent_events_targets.extend( bin[4]) #(0,1) without duration 0000111110000111110000 ec_independent_targets.extend( bin[5]) #(0,1,2,3,4) without duration 0000111110000222220000 ec_independent_event_ids.extend( bin[6] ) #(-1,0,1,2,3,4) without duration -1-1-1-100000-1-1-1-111111-1-1-1-1 tweets_length = torch.stack(bin[7]).to('cuda') tweets_length = tweets_length.squeeze(1) bin_tweets, tweets_length = self.sort_batch( bin_tweets, tweets_length) embeds = self.word_embeddings(bin_tweets).permute(1, 0, 2) if self.config.use_dropout == True: embeds = self.dropoutEmbeddings(embeds) embeds = pack_padded_sequence(embeds, tweets_length) # unpad if self.config.tweet_representation == "embeddings": tweet_representation = embeds tweet_representation, _ = pad_packed_sequence( tweet_representation) elif self.config.tweet_representation == "lstm": lstm_output, _ = self.lstm(embeds) tweet_representation, _ = pad_packed_sequence(lstm_output) if self.config.bin_representation == "avg": tl_expanded = (tweets_length).unsqueeze(1) avg_pool_byhand = torch.sum( tweet_representation, dim=0) / tl_expanded.to(dtype=torch.float) bin_representation = avg_pool_byhand elif self.config.bin_representation == "max": max_pool_byhand = torch.cat([ torch.max(i[:l], dim=0)[0].view(1, -1) for i, l in zip( tweet_representation.permute(1, 0, 2), tweets_length) ], dim=0) bin_representation = max_pool_byhand bin_representation = bin_representation.view( bin_representation.size(0), 1, bin_representation.size(1)) if self.config.bin_features == "avg": avg_pool = F.adaptive_avg_pool1d( bin_representation.permute(1, 2, 0), 1) #.view(1,1, -1) avg_pool.squeeze_(2).unsqueeze_(0) # 1,1,32 bin_features = avg_pool elif self.config.bin_features == "max": max_pool = F.adaptive_max_pool1d( bin_representation.permute(1, 2, 0), 1) #.view(1,1, -1) max_pool.squeeze_(2).unsqueeze_(0) # 1,1,32 bin_features = max_pool elif self.config.bin_features == "tweet_attention": bin_representation.squeeze_(1) scores = self.hidden2score(bin_representation) normalized_scores = self.softmax(scores) a_e = normalized_scores * bin_representation attented_representation = torch.sum(a_e, dim=0) bin_features = attented_representation #.view(1,1, -1) bin_features.unsqueeze_(0).unsqueeze_(0) elif self.config.bin_features == "tweet_attention2": bin_representation.squeeze_(1) inputs = bin_representation scores = self.relu(inputs.matmul(self.attention_weights2)) scores = self.softmax3(scores) weighted = torch.mul(inputs, scores.unsqueeze(-1).expand_as(inputs)) representations = weighted.sum(0).squeeze() bin_features = representations #.view(1, 1, -1) bin_features.unsqueeze_(0).unsqueeze_(0) elif self.config.bin_features == "cnn": embedded = bin_representation.squeeze(1).unsqueeze_( 0).unsqueeze_(0) conved = [ F.relu(conv(embedded)).squeeze(3) for conv in self.convs ] if self.config.cnn_pool == "max": pooled = [ F.max_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved ] elif self.config.cnn_pool == "avg": pooled = [ F.avg_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved ] bin_features = torch.cat(pooled, dim=1) if self.config.use_dropout == True: bin_features = self.cnn_dropout(torch.cat(pooled, dim=1)) bin_features.unsqueeze_(0) if self.config.non_linearity_bin_features == "relu": bin_features = self.relu(bin_features) elif self.config.non_linearity_bin_features == "tanh": bin_features = self.tanh(bin_features) lst_encodings.append(bin_features) lst_encodings = torch.cat(lst_encodings) # .to(self.device) if self.config.batch_norm == True: lst_encodings = self.dense1_bn( lst_encodings.view(1, lst_encodings.size(2), lst_encodings.size(0))) lst_encodings = lst_encodings.view(lst_encodings.size(2), 1, lst_encodings.size(1)) if self.config.use_BIO_LSTM == True: lstm_out2, _ = self.lstm2(lst_encodings) if self.config.use_dropout == True: lstm_out2 = self.dropoutLSTM2out(lstm_out2) match_representation = lstm_out2 else: match_representation = lst_encodings match_representation = match_representation.squeeze(1) tag_space = self.hidden2tag(match_representation) return tag_space, bio_targets, ec_targets, event_duration_targets, independent_events_targets, ec_independent_targets, ec_independent_event_ids
def forward(self, input): ''' 010 110 110 | conv layer | Lstm layer | reparameterization(optional) INPUT: Batch_size x (V) extra process x Sentence_Num x Sentence_Max_Len ''' # truncate kernel for saving resource batch_size = input.shape[0] sentence_num = input.shape[1] if len(input.shape) > 2: sentence_size = input.shape[2] # else: # sentence_num = 10 if sentence_num>10 # input = input.contiguous().view(batch_size, sentence_num, -1) time_1 = time.time() input, weight, compress_size = getattr(self, 'ver{}'.format(self.id))(input) time_1 = time.time() - time_1 if self.use_RNN: time_2 = time.time() self.cnn_out = [] for i in range(self.cnn_num): out = self.same_pad_conv2d(input, weight[i], self.bias[i], self.stride[i]) out = self.relu(out) # out = F.adaptive_max_pool2d(out, output_size = (sentence_num, 1)).squeeze(-1) # out = out.permute(0,2,1) # output: batch, sentence_num, input size self.cnn_out.append(out) concat_out = torch.cat(self.cnn_out, dim=1).permute(0, 2, 3, 1) word_att = self.word_attention(concat_out) time_2 = time.time() - time_2 time_3 = time.time() gru_out, h_n = self.gru(word_att) # gru_out.unsqueeze(2) sen_att = self.sen_attention(gru_out) output = self.fc(sen_att).squeeze(-1) time_3 = time.time() - time_3 self.tot_time += time_1 + time_2 + time_3 # print("1:{}, 2:{}, 3:{}, tot:{}".format(time_1/tot_time,time_2/tot_time,time_3/tot_time,tot_time)) else: input = input.view(batch_size, compress_size, 1, -1) self.cnn_out = [] for i in range(self.cnn_num): out = self.F.conv2d(input, weight[i], self.bias[i]).squeeze(-2) out = self.relu(out) out = F.adaptive_max_pool1d(out, output_size=1).squeeze(-1) self.cnn_out.append(out) combi_out = torch.cat(self.cnn_out, dim=1) output = self.fc1(combi_out).squeeze(-1) return output
def forward(self, q_vec, d_vec, sd_vec, labels=None): # embedding input vector q_emb = self.embedding(q_vec) d_emb = self.embedding(d_vec) sd_emb = self.embedding(sd_vec) q_transform = self.q_transformer(q_emb) d_transform = self.d_transformer(d_emb) sd_transform = self.sd_transformer(sd_emb) # Residual Net q_res = torch.cat((q_transform, q_emb), 2) d_res = torch.cat((d_transform, d_emb), 2) sd_res = torch.cat((sd_transform, sd_emb), 2) # alignment q_d_similarity = self.similarity(d_res, q_res) d2q = self.context_to_query(q_d_similarity, d_res) q2d = self.query_to_context(q_d_similarity, q_res) q_d_final = self.final_attention(d_res, d2q, q2d) q_sd_similarity = self.similarity(sd_res, q_res) sd2q = self.context_to_query(q_sd_similarity, sd_res) q2sd = self.query_to_context(q_sd_similarity, q_res) q_sd_final = self.final_attention(sd_res, sd2q, q2sd) q_d_concat = torch.cat((q_res, q_d_final, d_res), 2) # [batch_size, 128, embedding_size*12] q_sd_concat = torch.cat((q_res, q_sd_final, sd_res), 2) # [batch_size, 128, embedding_size*12] q_d_linear = self.q_d_Linear( q_d_concat) # [batch_size, 128, embedding_size*12] q_sd_linear = self.q_sd_Linear( q_sd_concat) # [batch_size, 128, embedding_size*12] all_concat_input = torch.cat((q_d_linear, q_sd_linear), 2) # [batch_size, 128, embedding_size*24] all_concat_input = all_concat_input.to(self.device) result_output = self.result_Linear( all_concat_input) # [batch_size, 128, embedding_size*24] result_output = result_output.to(self.device) result_output = result_output.permute(0, 2, 1) avg_pool = F.adaptive_avg_pool1d(result_output, 1) max_pool = F.adaptive_max_pool1d(result_output, 1) avg_pool = avg_pool.view(q_vec.size(0), -1) max_pool = max_pool.view(q_vec.size(0), -1) result = torch.cat((avg_pool, max_pool), 1) # [batch_size, embedding_size*48] logits = self.classifier(result) if labels is not None: loss_fct = CrossEntropyLoss() loss = loss_fct(logits, labels) return loss, logits else: return logits
def forward(self, x): x = F.adaptive_max_pool1d(x, output_size=7) x = F.adaptive_max_pool1d(x, output_size=1) return x
def forward(self, x): x = self.features(x) # print(x.size()) b = x.size(0) x = F.adaptive_max_pool1d(x, 1).view(b, 512) return x
def forward(self, x): batch_size = x.size(0) x = get_graph_feature( x, k=self.k ) # (batch_size, 3, num_points) -> (batch_size, 3*2, num_points, k) x = self.conv1( x ) # (batch_size, 3*2, num_points, k) -> (batch_size, 64, num_points, k) x1 = x.max( dim=-1, keepdim=False )[0] # (batch_size, 64, num_points, k) -> (batch_size, 64, num_points) x = get_graph_feature( x1, k=self.k ) # (batch_size, 64, num_points) -> (batch_size, 64*2, num_points, k) x = self.conv2( x ) # (batch_size, 64*2, num_points, k) -> (batch_size, 64, num_points, k) x2 = x.max( dim=-1, keepdim=False )[0] # (batch_size, 64, num_points, k) -> (batch_size, 64, num_points) x = get_graph_feature( x2, k=self.k ) # (batch_size, 64, num_points) -> (batch_size, 64*2, num_points, k) x = self.conv3( x ) # (batch_size, 64*2, num_points, k) -> (batch_size, 128, num_points, k) x3 = x.max( dim=-1, keepdim=False )[0] # (batch_size, 128, num_points, k) -> (batch_size, 128, num_points) x = get_graph_feature( x3, k=self.k ) # (batch_size, 128, num_points) -> (batch_size, 128*2, num_points, k) x = self.conv4( x ) # (batch_size, 128*2, num_points, k) -> (batch_size, 256, num_points, k) x4 = x.max( dim=-1, keepdim=False )[0] # (batch_size, 256, num_points, k) -> (batch_size, 256, num_points) x = torch.cat((x1, x2, x3, x4), dim=1) # (batch_size, 64+64+128+256, num_points) x = self.conv5( x ) # (batch_size, 64+64+128+256, num_points) -> (batch_size, emb_dims, num_points) x1 = F.adaptive_max_pool1d(x, 1).view( batch_size, -1) # (batch_size, emb_dims, num_points) -> (batch_size, emb_dims) x2 = F.adaptive_avg_pool1d(x, 1).view( batch_size, -1) # (batch_size, emb_dims, num_points) -> (batch_size, emb_dims) x = torch.cat((x1, x2), 1) # (batch_size, emb_dims*2) x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2 ) # (batch_size, emb_dims*2) -> (batch_size, 512) x = self.dp1(x) x = F.leaky_relu( self.bn7(self.linear2(x)), negative_slope=0.2) # (batch_size, 512) -> (batch_size, 256) x = self.dp2(x) x = self.linear3( x) # (batch_size, 256) -> (batch_size, output_channels) return x
def forward(self, pointcloud: torch.cuda.FloatTensor, cls): # x: B,3,N xyz, features = self._break_up_pc(pointcloud) num_pts = xyz.size(1) batch_size = xyz.size(0) # FPS to find different point subsets and their relations subset1_idx = pointnet2_utils.furthest_point_sample(xyz, num_pts // 4).long() # B,N/2 subset1_xyz = torch.unsqueeze(subset1_idx, -1).repeat(1, 1, 3) # B,N/2,3 subset1_xyz = torch.take(xyz, subset1_xyz) # B,N/2,3 dist, idx1 = pointnet2_utils.three_nn(xyz, subset1_xyz) dist_recip = 1.0 / (dist + 1e-8) norm = torch.sum(dist_recip, dim=2, keepdim=True) weight1 = dist_recip / norm subset12_idx = pointnet2_utils.furthest_point_sample( subset1_xyz, num_pts // 16).long() # B,N/4 subset12_xyz = torch.unsqueeze(subset12_idx, -1).repeat(1, 1, 3) # B,N/4,3 subset12_xyz = torch.take(subset1_xyz, subset12_xyz) # B,N/4,3 dist, idx12 = pointnet2_utils.three_nn(subset1_xyz, subset12_xyz) dist_recip = 1.0 / (dist + 1e-8) norm = torch.sum(dist_recip, dim=2, keepdim=True) weight12 = dist_recip / norm device = torch.device('cuda') centroid = torch.zeros([batch_size, 1, 3], device=device) dist, idx0 = pointnet2_utils.three_nn(subset12_xyz, centroid) dist_recip = 1.0 / (dist + 1e-8) norm = torch.sum(dist_recip, dim=2, keepdim=True) weight0 = dist_recip / norm ####################################### # Error-minimizing module 1: # Encoding x = xyz.transpose(2, 1) # x: B,3,N x1_1 = x x = get_adptive_dilated_graph_feature(x, self.conv_op1, self.conv_op11, self.conv_op12, d=5, k=20) x = self.conv1(x) # B,64,N,k x = self.conv14(x) # B,64,N,k x1_2 = x # Back-projection x = self.conv11(x) # B,3,N,1 x = torch.squeeze(x, -1) # B,3,N x1_3 = x # Calculating Error delta_1 = x1_3 - x1_1 # B,3,N # Output x = x1_2 # B,64,N,k x1 = x.max(dim=-1, keepdim=False)[0] # B,64,N ####################################### ####################################### # Multi-resolution (MR) Branch # Down-scaling 1 subset1_feat = torch.unsqueeze(subset1_idx, -1).repeat(1, 1, 64) # B,N/2,64 x1_subset1 = torch.take(x1.transpose(1, 2).contiguous(), subset1_feat).transpose( 1, 2).contiguous() # B,64,N/2 x2_1 = x1_subset1 # B,64,N/2 x = get_graph_feature(x1_subset1, k=self.k // 2) x = self.conv2(x) # B,64,N/2,k x = self.conv24(x) # B,128,N/2,k x2 = x.max(dim=-1, keepdim=False)[0] # B,128,N/2 # Dense-connection x12 = pointnet2_utils.three_interpolate(x2, idx1, weight1) # B,128,N x12 = torch.cat((x12, x1), dim=1) # B,192,N x12 = self.conv23(x12) # B,128,N # Down-scaling 2 subset12_feat = torch.unsqueeze(subset12_idx, -1).repeat(1, 1, 128) # B,N/4,128 x2_subset12 = torch.take( x2.transpose(1, 2).contiguous(), subset12_feat).transpose(1, 2).contiguous() # B,128,N/4 x3_1 = x2_subset12 # B,128,N/4 x = get_graph_feature(x2_subset12, k=self.k // 4) x = self.conv3(x) # B,256,N/4,k x3 = x.max(dim=-1, keepdim=False)[0] # B,256,N/4 # Dense-connection x23 = pointnet2_utils.three_interpolate(x3, idx12, weight12) # B,256,N/2 x23 = torch.cat((x23, x2), dim=1) # B,384,N/2 x23 = self.conv34(x23) # B,128,N/2 x123 = pointnet2_utils.three_interpolate(x23, idx1, weight1) # B,128,N x123 = torch.cat((x123, x12, x1), dim=1) # B,320,N x123 = self.conv35(x123) # B,128,N # Down-scaling 3 x_bot = self.conv53(x3) x_bot = self.conv54(x_bot) # B,1024,N/128 x_bot = F.adaptive_max_pool1d(x_bot, 1) # B,1024,1 # Upsampling 3: interpolated_feats1 = pointnet2_utils.three_interpolate( x_bot, idx0, weight0) # B,1024,N/4 interpolated_feats2 = x3 # B,256,N/4 x3_up = torch.cat((interpolated_feats1, interpolated_feats2), dim=1) # B,1280,N/4 x3_up = self.conv32(x3_up) # B,256,N/4 x3_up = self.conv33(x3_up) # B,256,N/4 # Upsampling 2: interpolated_feats1 = pointnet2_utils.three_interpolate( x3_up, idx12, weight12) # B,256,N/2 interpolated_feats2 = x2 # B,128,N/2 interpolated_feats3 = x23 # B,128,N/2 x2_up = torch.cat( (interpolated_feats1, interpolated_feats3, interpolated_feats2), dim=1) # B,512,N/2 x2_up = self.conv21(x2_up) # B,256,N/2 x2_up = self.conv22(x2_up) # B,128,N/2 # Upsampling 1: interpolated_feats1 = pointnet2_utils.three_interpolate( x2_up, idx1, weight1) # B,128,N interpolated_feats2 = x1 # B,64,N interpolated_feats3 = x12 # B,128,N interpolated_feats4 = x123 # B,128,N x1_up = torch.cat((interpolated_feats1, interpolated_feats4, interpolated_feats3, interpolated_feats2), dim=1) # B,448,N x1_up = self.conv12(x1_up) # B,512,N x1_up = self.conv13(x1_up) # B,1024,N x_mr = x1_up ############################################################################# ############################################################################# # Full-resolution Branch # Error-minimizing module 2: # Encoding x2_1 = x1 # B,64,N x = get_adptive_dilated_graph_feature(x1, self.conv_op2, self.conv_op21, self.conv_op22, d=5, k=20) x = self.convfc2(x) # B,64,N,k x = self.convfc24(x) # B,64,N,k x2_2 = x # Back-projection x = self.convfc21(x) # B,64,N,1 x = torch.squeeze(x, -1) # B,64,N x2_3 = x # Calculating Error delta_2 = x2_3 - x2_1 # B,64,N # Output x = x2_2 # B,64,N,k x2 = x.max(dim=-1, keepdim=False)[0] # B,64,N ####################################### # Error-minimizing module 3: # Encoding x3_1 = x2 # B,64,N x = get_adptive_dilated_graph_feature(x2, self.conv_op3, self.conv_op31, self.conv_op32, d=5, k=20) x = self.convfc3(x) # B,128,N,k x3_2 = x # Back-projection x = self.convfc31(x) # B,64,N,1 x = torch.squeeze(x, -1) # B,64,N x3_3 = x # Calculating Error delta_3 = x3_3 - x3_1 # B,64,N # Output x = x3_2 # B,128,N,k x3 = x.max(dim=-1, keepdim=False)[0] # B,128,N ####################################### # Error-minimizing module 4: # Encoding x4_1 = x3 # B,128,N x = get_adptive_dilated_graph_feature(x3, self.conv_op4, self.conv_op41, self.conv_op42, d=5, k=20) x = self.convfc4(x) # B,256,N,k x4_2 = x # Back-projection x = self.convfc41(x) # B,128,N,1 x = torch.squeeze(x, -1) # B,128,N x4_3 = x # Calculating Error delta_4 = x4_3 - x4_1 # B,128,N # Output x = x4_2 # B,256,N,k x4 = x.max(dim=-1, keepdim=False)[0] # B,256,N x = torch.cat((x1, x2, x3, x4), dim=1) # B,512,N x_fr = self.conv7(x) # B,1024,N # Fusing FR and MR outputs fusion_score = self.fuse(x_mr) x = x_fr + x_fr * fusion_score x_all = self.conv9(x) # B,1024,N # Collecting global feature one_hot_label = cls.view(-1, 16, 1) # B,16,1 one_hot_label = self.conv5(one_hot_label) # B,64,1 x_max = F.adaptive_max_pool1d(x_all, 1) # B,1024,1 x_global = torch.cat((x_max, one_hot_label), dim=1) # B,1088,1 x_global = x_global.repeat(1, 1, num_pts) # B,1088,N x = torch.cat((x_all, x_global), dim=1) # B,2112,N x = self.conv8(x) # B,1024,N x = self.conv63(x) # B,128,N x = self.dp(x) x = self.conv64(x) # B,50,N return (x.transpose(2, 1).contiguous(), delta_1.transpose(2, 1).contiguous(), delta_2.transpose(2, 1).contiguous(), delta_3.transpose(2, 1).contiguous(), delta_4.transpose(2, 1).contiguous())
def test_adaptive_max_pool1d(self): inp = torch.randn(1, 16, 28, device='cuda', dtype=self.dtype) out = F.adaptive_max_pool1d(inp, output_size=5, return_indices=True)
def forward(self, x1_seq, len1, x2_seq, len2, pooling_mode='hstates', device="cpu", output_state_vectors=False, evaluation=False): if evaluation: # XXX Set dropouts to zero manually self.att1_dropout = 0 self.att2_dropout = 0 self.fc1_dropout = 0 self.fc2_dropout = 0 if output_state_vectors: create_parent_dir(output_state_vectors) self.h1, self.c1 = self.init_hidden(x1_seq.size(1), device) x1_embs_not_packed = self.emb(x1_seq) x1_embs = pack_padded_sequence(x1_embs_not_packed, len1, enforce_sorted=False) # To avoid the following issue: # RNN module weights are not part of single contiguous chunk of memory. # This means they need to be compacted at every call, possibly greatly increasing memory usage. # To compact weights again call flatten_parameters(). self.rnn_1.flatten_parameters() if self.main_architecture.lower() in ["lstm"]: rnn_out_1, (self.h1, self.c1) = self.rnn_1(x1_embs, (self.h1, self.c1)) elif self.main_architecture.lower() in ["gru", "rnn"]: rnn_out_1, self.h1 = self.rnn_1(x1_embs, self.h1) rnn_out_1, len1 = pad_packed_sequence(rnn_out_1) if output_state_vectors: # the layers can be separated using h_n.view(num_layers, num_directions, batch, hidden_size). h1_reshape = self.h1.view(self.rnn_n_layers, self.num_directions, rnn_out_1.shape[1], self.rnn_hidden_dim) output_h_layer = self.rnn_n_layers - 1 if not self.file_id: self.file_id = len(glob.glob(output_state_vectors + "_fwd_*")) else: self.file_id += 1 torch.save(h1_reshape[output_h_layer, 0], f'{output_state_vectors}_fwd_{self.file_id}') if self.bidirectional: torch.save(h1_reshape[output_h_layer, 1], f'{output_state_vectors}_bwd_{self.file_id}') return (h1_reshape[output_h_layer, 0], h1_reshape[output_h_layer, 1]) else: return (h1_reshape[output_h_layer, 0], False) if pooling_mode in ['attention']: attn_weight_flag = False attn_weight_array = False for i_nhs in range(np.shape(rnn_out_1)[0]): attn_weight = F.relu(self.attn_step1(F.dropout(rnn_out_1[i_nhs], self.att1_dropout))) attn_weight = self.attn_step2(F.dropout(attn_weight, self.att2_dropout)) if not attn_weight_flag: attn_weight_array = attn_weight attn_weight_flag = True else: attn_weight_array = torch.cat((attn_weight_array, attn_weight), dim=1) attn_weight_array = F.softmax(attn_weight_array, dim=1) attn_vect_1 = torch.squeeze(torch.bmm(rnn_out_1.permute(1, 2, 0), torch.unsqueeze(attn_weight_array, 2))) elif pooling_mode in ['average']: pool_1 = F.adaptive_avg_pool1d(rnn_out_1.permute(1, 2, 0), 1).view(x1_seq.size(1), -1) elif pooling_mode in ['max', 'maximum']: pool_1 = F.adaptive_max_pool1d(rnn_out_1.permute(1, 2, 0), 1).view(x1_seq.size(1), -1) elif pooling_mode in ['hstates']: hstates_1_fwd_bwd = self.h1.view(self.rnn_n_layers, self.num_directions, rnn_out_1.shape[1], self.rnn_hidden_dim) hstates_1 = hstates_1_fwd_bwd[self.rnn_n_layers - 1, 0] if self.bidirectional: hstates_1 = torch.cat((hstates_1, hstates_1_fwd_bwd[self.rnn_n_layers - 1, 1]), dim=1) elif pooling_mode in ['hstates_layers', 'hstates_layers_simple', 'hstates_subtract', 'hstates_l2_distance', 'hstates_cosine']: hstates_1_fwd_bwd = self.h1.view(self.rnn_n_layers, self.num_directions, rnn_out_1.shape[1], self.rnn_hidden_dim) hstates_1 = hstates_1_fwd_bwd[0, 0] for rlayer in range(1, self.rnn_n_layers): hstates_1 = torch.cat((hstates_1, hstates_1_fwd_bwd[rlayer, 0]), dim=1) if self.bidirectional: hstates_1_bwd = hstates_1_fwd_bwd[0, 1] for rlayer in range(1, self.rnn_n_layers): hstates_1_bwd = torch.cat((hstates_1_bwd, hstates_1_fwd_bwd[rlayer, 1]), dim=1) hstates_1 = torch.cat((hstates_1, hstates_1_bwd), dim=1) self.h2, self.c2 = self.init_hidden(x2_seq.size(1), device) x2_embs_not_packed = self.emb(x2_seq) x2_embs = pack_padded_sequence(x2_embs_not_packed, len2, enforce_sorted=False) # Share parameters between two GRUs # Previously, we had gru_out_2, self.h2 = self.gru_2(x2_embs, self.h2) if self.main_architecture.lower() in ["lstm"]: rnn_out_2, (self.h2, self.c2) = self.rnn_1(x2_embs, (self.h2, self.c2)) elif self.main_architecture.lower() in ["gru", "rnn"]: rnn_out_2, self.h2 = self.rnn_1(x2_embs, self.h2) rnn_out_2, len2 = pad_packed_sequence(rnn_out_2) if pooling_mode in ['attention']: attn_weight_flag = False attn_weight_array = False for i_nhs in range(np.shape(rnn_out_2)[0]): attn_weight = F.relu(self.attn_step1(F.dropout(rnn_out_2[i_nhs], self.att1_dropout))) attn_weight = self.attn_step2(F.dropout(attn_weight, self.att2_dropout)) if not attn_weight_flag: attn_weight_array = attn_weight attn_weight_flag = True else: attn_weight_array = torch.cat((attn_weight_array, attn_weight), dim=1) attn_weight_array = F.softmax(attn_weight_array, dim=1) attn_vect_2 = torch.squeeze(torch.bmm(rnn_out_2.permute(1, 2, 0), torch.unsqueeze(attn_weight_array, 2))) elif pooling_mode in ['average']: pool_2 = F.adaptive_avg_pool1d(rnn_out_2.permute(1, 2, 0), 1).view(x2_seq.size(1), -1) elif pooling_mode in ['max', 'maximum']: pool_2 = F.adaptive_max_pool1d(rnn_out_2.permute(1, 2, 0), 1).view(x2_seq.size(1), -1) elif pooling_mode in ['hstates']: hstates_2_fwd_bwd = self.h2.view(self.rnn_n_layers, self.num_directions, rnn_out_2.shape[1], self.rnn_hidden_dim) hstates_2 = hstates_2_fwd_bwd[self.rnn_n_layers - 1, 0] if self.bidirectional: hstates_2 = torch.cat((hstates_2, hstates_2_fwd_bwd[self.rnn_n_layers - 1, 1]), dim=1) elif pooling_mode in ['hstates_layers', 'hstates_layers_simple', 'hstates_subtract', 'hstates_l2_distance', 'hstates_cosine']: hstates_2_fwd_bwd = self.h2.view(self.rnn_n_layers, self.num_directions, rnn_out_2.shape[1], self.rnn_hidden_dim) hstates_2 = hstates_2_fwd_bwd[0, 0] for rlayer in range(1, self.rnn_n_layers): hstates_2 = torch.cat((hstates_2, hstates_2_fwd_bwd[rlayer, 0]), dim=1) if self.bidirectional: hstates_2_bwd = hstates_2_fwd_bwd[0, 1] for rlayer in range(1, self.rnn_n_layers): hstates_2_bwd = torch.cat((hstates_2_bwd, hstates_2_fwd_bwd[rlayer, 1]), dim=1) hstates_2 = torch.cat((hstates_2, hstates_2_bwd), dim=1) # Combine outputs from GRU1 and GRU2 if pooling_mode in ['attention']: attn_vec_cat = torch.cat((attn_vect_1, attn_vect_2), dim=1) attn_vec_mul = attn_vect_1 * attn_vect_2 attn_vec_dif = attn_vect_1 - attn_vect_2 output_combined = torch.cat((attn_vec_cat, attn_vec_mul, attn_vec_dif), dim=1) elif pooling_mode in ['average', 'max', 'maximum']: pool_rnn_cat = torch.cat((pool_1, pool_2), dim=1) pool_rnn_mul = pool_1 * pool_2 pool_rnn_dif = pool_1 - pool_2 output_combined = torch.cat((pool_rnn_cat, pool_rnn_mul, pool_rnn_dif), dim=1) elif pooling_mode in ['hstates', 'hstates_layers']: hstates_rnn_cat = torch.cat((hstates_1, hstates_2), dim=1) hstates_rnn_mul = hstates_1 * hstates_2 hstates_rnn_dif = hstates_1 - hstates_2 output_combined = torch.cat((hstates_rnn_cat, hstates_rnn_mul, hstates_rnn_dif), dim=1) elif pooling_mode in ['hstates_layers_simple']: output_combined = torch.cat((hstates_1, hstates_2), dim=1) elif pooling_mode in ['hstates_subtract']: output_combined = 1 - torch.abs(hstates_1 - hstates_2) elif pooling_mode in ['hstates_l2_distance']: output_combined = 1 - torch.abs(hstates_1 - hstates_2)**2 elif pooling_mode in ['hstates_cosine']: hstates_cosine_sim = nn.CosineSimilarity(dim=1, eps=1e-10)(hstates_1, hstates_2) # in this case, return the cosine similarity as predictions #return torch.log(torch.stack([1- hstates_cosine_sim, hstates_cosine_sim])).T return torch.stack([1-hstates_cosine_sim, hstates_cosine_sim]).T y_out = F.relu(self.fc1(F.dropout(output_combined, self.fc1_dropout))) y_out = self.fc2(F.dropout(y_out, self.fc2_dropout)) return y_out
def forward(self, x, phase='train'): out = self.conv(x) # print out.size() out = self.bn(out) # print out.size() out = self.relu(out) # print out.size() out1 = self.layer1(out) # 64 # print out1.size() out2 = self.layer2(out1) # 32 # print out2.size() out3 = self.layer3(out2) # 16 # print out3.size() out4 = self.layer4(out3) # 8 # print out4.size() out5 = self.layer5(out4) # 4 # print out5.size() # out = F.adaptive_max_pool2d(out5, output_size=(1,1)).view(out.size(0), -1) # 128 # out = out.view(out.size(0), -1) if phase == 'seg': out = F.adaptive_max_pool2d(out5, output_size=(1, 1)).view(out.size(0), -1) # 128 out = self.fc(out) out = out.view(out.size(0), -1) else: out = F.max_pool2d(out5, 2) out_size = out.size() # out = out.view(out_size[0],out_size[1],out_size[3]).transpose(1,2).contiguous().view(-1, out_size[1]) out = out.view(out_size[0], out_size[1], out_size[2] * out_size[3]).transpose( 1, 2).contiguous().view(-1, out_size[1]) out = self.fc(out) out = out.view(out_size[0], out_size[2] * out_size[3], -1).transpose(1, 2).contiguous() out = F.adaptive_max_pool1d(out, output_size=(1)).view(out_size[0], -1) # print out.size() if phase not in ['seg', 'pretrain', 'pretrain2']: return out # detect cat1 = torch.cat([self.convt1(out5), out4], 1) # print cat1.size() dec1 = self.dec1(cat1) # print dec1.size() # print out3.size() cat2 = torch.cat([self.convt2(dec1), out3], 1) # print cat2.size() dec2 = self.dec2(cat2) cat3 = torch.cat([self.convt3(dec2), out2], 1) dec3 = self.dec3(cat3) cat4 = torch.cat([self.convt4(dec3), out1], 1) seg = self.dec4(cat4) seg = seg.view((seg.size(0), seg.size(2), seg.size(3))) seg = self.sigmoid(seg) bbox = self.bbox(cat2) # dec2 = self.output(dec2) # print dec2.size() size = bbox.size() bbox = bbox.view((size[0], size[1], -1)).transpose(1, 2).contiguous() bbox = bbox.view((size[0], size[2], size[3], -1, 4)) return out, bbox, seg
def forward(self, sent_m, mention_m, relations_m, relations_words_m, entities_m, candidates_m, candidates_labels_m, features_m): choices = candidates_labels_m.size( 1) # number of possible candidates per one mention real_choices_num = torch.sum((candidates_m > 0).float(), dim=1).unsqueeze(1) sent_emb = self._words2vector(sent_m).unsqueeze(1) mention_emb = self._chars2vector(mention_m).unsqueeze(1) sent_emb_expanded = sent_emb.expand(sent_emb.size(0), choices, sent_emb.size(2)).contiguous() mention_emb_expanded = mention_emb.expand( mention_emb.size(0), choices, mention_emb.size(2)).contiguous() relations_words_m = relations_words_m.view( relations_words_m.size(0) * relations_words_m.size(1) * relations_words_m.size(2), -1) relations_m = relations_m.view( relations_m.size(0) * relations_m.size(1), -1) entities_m = entities_m.view( entities_m.size(0) * entities_m.size(1), -1) candidates_labels_m = candidates_labels_m.view( candidates_labels_m.size(0) * candidates_labels_m.size(1), -1, 2) candidate_vectors = self.compute_candidate_vectors( relations_m, relations_words_m, entities_m, candidates_m, candidates_labels_m) candidate_vectors = candidate_vectors.view(-1, choices, candidate_vectors.size(-1)) features_m = features_m.float() concatenated_embed = torch.cat( (sent_emb_expanded, mention_emb_expanded, candidate_vectors, features_m), dim=-1).contiguous() concatenated_embed = concatenated_embed.view( -1, concatenated_embed.size(-1)) sem_vector = self.sem_layers(concatenated_embed) sem_vector = sem_vector.view(-1, choices, sem_vector.size(-1)) sem_vector_pooled_over_choices = sem_vector.transpose(-2, -1) sem_vector_pooled_over_choices = self._pool( sem_vector_pooled_over_choices) sem_vector_pooled_over_choices = sem_vector_pooled_over_choices.transpose( -2, -1) sem_vector = torch.cat( (sem_vector, sem_vector_pooled_over_choices.expand_as(sem_vector)), dim=-1) sem_vector = sem_vector.view(-1, sem_vector.size(-1)) candidate_scores = self.score_weights(sem_vector).squeeze(dim=-1) candidate_scores = candidate_scores.view(-1, choices) negative_vector = torch.cat( (sent_emb.squeeze(dim=1), mention_emb.squeeze(dim=1)), dim=1) negative_vector = self.negative_layers(negative_vector) real_choices_num = self._nonlinearity(real_choices_num) choices_pooled_for_negative = sem_vector_pooled_over_choices.squeeze( dim=1) negative_vector = torch.cat( (negative_vector, choices_pooled_for_negative, F.adaptive_max_pool1d(candidate_scores.unsqueeze(1), 1).squeeze(dim=-1), real_choices_num), dim=-1) negative_score = self.negative_weight(negative_vector) candidate_scores = self._nonlinearity(candidate_scores) return F.sigmoid(negative_score.squeeze(dim=-1)), candidate_scores
def forward(self, x): batch_size = x.size(0) # shape of x : batch, feature, npoints, neighbors # convolution(shared mlp to xi, xj-xi & max) # => # transformer(shared wq, wk, wv to xi) if self.ape: ptcld = x else: ptcld = None #print("1") x, abs_x = get_neighbors(x, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn1) # b, 3, 1024, 20 x1 = self.conv1(x, abs_x) # b, 64, 1024 x1 = self.relu(self.bn1(x1)).squeeze(3) #print("2") x, abs_x = get_neighbors(x1, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn2, ape=self.pos_nn2, ptcld=ptcld) # b, 64, 1024, 20 x2 = self.conv2(x, abs_x) # b, 64, 1024 x2 = self.relu(self.bn2(x2)).squeeze(3) #print("3") x, abs_x = get_neighbors(x2, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn3, ape=self.pos_nn3, ptcld=ptcld) # b, 128, 1024, 20 x3 = self.conv3(x, abs_x) # b, 128, 1024 x3 = self.relu(self.bn3(x3)).squeeze(3) #print("4") x, abs_x = get_neighbors(x3, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn4, ape=self.pos_nn4, ptcld=ptcld) # b, 256, 1024, 20 x4 = self.conv4(x, abs_x) # b, 256, 1024, 20 x4 = self.relu(self.bn4(x4)).squeeze(3) x = self.conv5(x4) x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1) x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2) x = self.dp1(x) x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2) x = self.dp2(x) x = self.linear3(x) return x
def aggregate(self, inputs): aggregated = func.adaptive_max_pool1d(inputs, 1) return aggregated
def pool(self, input): return F.adaptive_max_pool1d(input, 1)
def forward(self, x): batch_size = x.size(0) num_points_1 = x.size(2) # num_points_2 = int(num_points_1/2) # num_points_3 = int(num_points_1/4) num_points_2 = int(num_points_1 * 0.75) num_points_3 = int(num_points_2 * 0.75) num_points_4 = int(num_points_3 * 0.75) num_points_5 = int(num_points_4 * 0.75) ########################BLOCK1############################## N1_points = x.permute(0, 2, 1).contiguous() x1 = self.GSCM(N1_points, None, self.k, self.conv1, isFirstLayer=True) ########################BLOCK2############################## fps_id_2 = pointnet2_utils.furthest_point_sample( N1_points, num_points_2) N2_points = (pointnet2_utils.gather_operation( N1_points.transpose(1, 2).contiguous(), fps_id_2).transpose(1, 2).contiguous()) x1_downSample = (pointnet2_utils.gather_operation(x1, fps_id_2)) x2 = self.GSCM(N2_points, x1_downSample, self.k, self.conv2) ########################BLOCK3############################## fps_id_3 = pointnet2_utils.furthest_point_sample( N2_points, num_points_3) N3_points = (pointnet2_utils.gather_operation( N2_points.transpose(1, 2).contiguous(), fps_id_3).transpose(1, 2).contiguous()) x2_downSample = (pointnet2_utils.gather_operation(x2, fps_id_3)) x1_downSample = (pointnet2_utils.gather_operation( x1_downSample, fps_id_3)) x3 = self.GSCM(N3_points, x2_downSample, self.k, self.conv3) ########################BLOCK4############################## fps_id_4 = pointnet2_utils.furthest_point_sample( N3_points, num_points_4) N4_points = (pointnet2_utils.gather_operation( N3_points.transpose(1, 2).contiguous(), fps_id_4).transpose(1, 2).contiguous()) x3_downSample = (pointnet2_utils.gather_operation(x3, fps_id_4)) x2_downSample = (pointnet2_utils.gather_operation( x2_downSample, fps_id_4)) x1_downSample = (pointnet2_utils.gather_operation( x1_downSample, fps_id_4)) x4 = self.GSCM(N4_points, x3_downSample, self.k, self.myconv4) ########################BLOCK5############################## fps_id_5 = pointnet2_utils.furthest_point_sample( N4_points, num_points_5) N5_points = (pointnet2_utils.gather_operation( N4_points.transpose(1, 2).contiguous(), fps_id_5).transpose(1, 2).contiguous()) x4_downSample = (pointnet2_utils.gather_operation(x4, fps_id_5)) x3_downSample = (pointnet2_utils.gather_operation( x3_downSample, fps_id_5)) x2_downSample = (pointnet2_utils.gather_operation( x2_downSample, fps_id_5)) x1_downSample = (pointnet2_utils.gather_operation( x1_downSample, fps_id_5)) x5 = self.GSCM(N5_points, x4_downSample, self.k, self.myconv5) x = torch.cat( (x1_downSample, x2_downSample, x3_downSample, x4_downSample, x5), dim=1) x = self.conv5(x) x1 = F.adaptive_max_pool1d(x, 1).view(batch_size, -1) x2 = F.adaptive_avg_pool1d(x, 1).view(batch_size, -1) x = torch.cat((x1, x2), 1) x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2) x = self.dp1(x) x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2) x = self.dp2(x) x = self.linear3(x) return x
def forward(self, x): batch_size = x.size(0) # shape of x : batch, feature, npoints, neighbors # convolution(shared mlp to xi, xj-xi & max) # => # transformer(shared wq, wk, wv to xi) if self.ape: ptcld = x else: ptcld = None x, abs_x, deg1, idx1 = get_neighbors(x, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn1, layer=1) # b, 64, 1024, 20 #print("1") idx = idx1 x1, k1, v1 = self.conv1(x, abs_x, deg1, idx) # b, 64, 1024 x1 = self.act1(self.bn1(x1)).squeeze(3) k = k1 v = v1 #print("2") x, abs_x, deg2, idx2 = get_neighbors(x1, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn2, layer=2) # b, 64, 1024, 20 x2, k2, v2 = self.conv2(x, abs_x, deg2, idx2, k=k, v=v) # b, 64, 1024 x2 = self.act2(self.bn2(x2)).squeeze(3) k = torch.cat((k, k2), dim=3) # b, 64, 1024, 2 v = torch.cat((v, v2), dim=3) # b, 64, 1024, 2 #print("3") x, abs_x, deg3, idx3 = get_neighbors(x2, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn3, layer=3) # b, 64, 1024, 20 x3, k3, v3 = self.conv3(x, abs_x, deg3, idx3, k=k, v=v) # b, 128, 1024 x3 = self.act3(self.bn3(x3)).squeeze(3) k = torch.cat((k, k3), dim=3) # b, 64, 1024, 3 v = torch.cat((v, v3), dim=3) # b, 64, 1024, 3 #print("4") x, abs_x, deg4, idx4 = get_neighbors(x3, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn4, layer=4) # b, 64, 1024, 20 x4, k4, v4 = self.conv4(x, abs_x, deg4, idx4, k=k, v=v) # b, 256, 1024, 20 x4 = self.act4(self.bn4(x4)).squeeze(3) x = self.conv5(x4) x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1) x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2) x = self.dp1(x) x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2) x = self.dp2(x) x = self.linear3(x) return x
def forward(self, x): b = x.size(0) # batch size # d = x.size(1) # dimensionality of input n = x.size(2) # number of points # EdgeConv ------------------------------------ x = get_graph_feature(x, rsize=self.rsize, diff_features_only=self.diff_features_only ) # (b, d, n) -> (b, 2*d, n, F) x = self.dpR1(x) # (b, 2*d, n) -> (b, 2*d, n, F) x = self.convR1C1(x) # (b, 2*d, n, F) -> (b, 64, n, F) x = self.convR1C2(x) # (b, 64, n, F) -> (b, 64, n, F) r1 = x.max(dim=-1, keepdim=False)[0] # (b, 64, n, F) -> (b, 64, n) # --------------------------------------------- # EdgeConv ------------------------------------ x = get_graph_feature(r1, rsize=self.rsize, diff_features_only=self.diff_features_only ) # (b, 64, n) -> (b, 64*2, n, F) x = self.dpR2(x) # (b, 2*d, n) -> (b, 2*d, n, F) x = self.convR2C1(x) # (b, 64*2, n, F) -> (b, 64, n, F) x = self.convR2C2(x) # (b, 64, n, F) -> (b, 64, n, F) r2 = x.max(dim=-1, keepdim=False)[0] # (b, 64, n, F) -> (b, 64, n) # --------------------------------------------- x = torch.cat((r1, r2), 1) # (b, 128, n) x = self.convC1(x) # (b, 128, n) -> (b, 256, n) # GlobConv ------------------------------------ # GlobExt ---------- x_glob = self.convG11(x) # (b, 256, n) -> (b, 32, n) x_max = F.adaptive_max_pool1d(x_glob, 1) # (b, 32, n) -> (b, 32, 1) x_max = x_max.view(b, -1) # (b, 32, 1) -> (b, 32) x_avg = F.adaptive_avg_pool1d(x_glob, 1) # (b, 32, n) -> (b, 32, 1) x_avg = x_avg.view(b, -1) # (b, 32, 1) -> (b, 32) x_glob = torch.cat((x_max, x_avg), 1) # (b, 64) x_glob = self.convG12(x_glob).unsqueeze(-1) # (b, 64) -> (b, 32, 1) # ------------------ x = self.convC2(x) # (b, 256, n) -> (b, 96, n) x = torch.cat((x, x_glob.repeat(1, 1, n)), 1) # (b, 96, n) -> (b, 128, n) x = self.convC3(x) # (b, 128, n) -> (b, 64, n) # --------------------------------------------- x = self.dp(x) # (b, 64, n) -> (b, 64, n)) x = self.convOut(x) # (b, 64, n) -> (b, out_channels, n) return x
def forward(self, x): return torch.cat( (F.adaptive_avg_pool1d(x, 1), F.adaptive_max_pool1d(x, 1)), dim=1)
def forward(self, x): x = F.relu(self.bn1(self.conv1(x))) x = F.relu(self.bn2(self.conv2(x))) x = F.adaptive_max_pool1d(x, 1).squeeze() return x
def adaptive_catavgmax_pool1d(x, output_size=1): x_avg = F.adaptive_avg_pool1d(x, output_size) x_max = F.adaptive_max_pool1d(x, output_size) return torch.cat((x_avg, x_max), 1)
lstm = nn.LSTM(8, 10, 1, bidirectional=True) batch_size = input3.size(0) print(batch_size) h = Variable(torch.zeros((2, 2, 10))) c = Variable(torch.zeros((2, 2, 10))) lstm_out, (h, c) = lstm(output3, (h, c)) print( lstm_out, lstm_out.shape) #lstm out => (sequence len, batch size, hidden_dim*2) # (batch_size, hidden_dim*2, sequence_len) avg_pool = F.adaptive_avg_pool1d(lstm_out.permute(1, 2, 0), 1).view(batch_size, -1) max_pool = F.adaptive_max_pool1d(lstm_out.permute(1, 2, 0), 1).view(batch_size, -1) sentence_output = torch.cat( [c[-1], avg_pool, max_pool], dim=1) # (batch_size, 3*num_directions*n_hidden) print(sentence_output.shape) all_tog = torch.cat([output, sentence_output], dim=1) linear = nn.Linear(61, 1) out = linear(all_tog) print(out)