def compute_loss(self, output): """Since we compute the loss for every mini-batch, we first calculate the loss for each sample, sum it and divide it by the batch size. Arguments: output {torch.Tensor} -- output of the network Returns: [type] -- [description] """ if self.loss == 'bce': if self.training: return F.binary_cross_entropy_with_logits(output, self.train_gt, self.train_weight, reduction='sum') / config.train_batch_size else: return F.binary_cross_entropy_with_logits(output, self.valid_gt, self.valid_weight, reduction='sum') / config.train_batch_size else: if self.training: return F.soft_margin_loss(output, self.train_gt, reduction='sum') / config.train_batch_size else: return F.soft_margin_loss(output, self.valid_gt, reduction='sum') / config.train_batch_size
def get_addition_loss(ca_weights, sca_weights, sa_weights, ssa_weights, args): if args.addition_loss == 'norm_softmargin': total_loss = args.ca_trade_off * F.soft_margin_loss(ca_weights / torch.norm(ca_weights, p=2, dim=1, keepdim=True), sca_weights.detach() / torch.norm(sca_weights.detach(), p=2, dim=1, keepdim=True)) total_loss += args.sa_trade_off * F.soft_margin_loss(sa_weights / torch.norm(sa_weights, p=2, dim=(2,3), keepdim=True), ssa_weights.detach() / torch.norm(ssa_weights.detach(), p=2, dim=(2,3), keepdim=True)) elif args.addition_loss == 'softmargin': total_loss = args.ca_trade_off * F.soft_margin_loss(ca_weights, sca_weights.detach()) total_loss += args.sa_trade_off * F.soft_margin_loss(sa_weights, ssa_weights.detach()) return total_loss
def quadruplet_loss(src_embed, tgt_embed, tgt_label, alpha1, alpha2, norm_feat, hard_mining): # loss_triplet = triplet_loss(tgt_embed, tgt_label, alpha1, norm_feat, hard_mining) if norm_feat: tgt_dist_mat = cosine_dist(tgt_embed, tgt_embed) src_tgt_dist_mat = cosine_dist(src_embed, tgt_embed) else: tgt_dist_mat = euclidean_dist(tgt_embed, tgt_embed) src_tgt_dist_mat = euclidean_dist(src_embed, tgt_embed) N = tgt_dist_mat.size(0) is_pos = tgt_label.view(N, 1).expand(N, N).eq(tgt_label.view(N, 1).expand(N, N).t()).float() is_neg = tgt_label.view(N, 1).expand(N, N).ne(tgt_label.view(N, 1).expand(N, N).t()).float() # print(is_pos) if hard_mining: dist_ap, dist_an = hard_example_mining(tgt_dist_mat, is_pos, is_neg) _, dist_nn = hard_example_mining(src_tgt_dist_mat, torch.zeros_like(is_pos), torch.ones_like(is_neg)) else: dist_ap, dist_an = weighted_example_mining(tgt_dist_mat, is_pos, is_neg) _, dist_nn = weighted_example_mining(src_tgt_dist_mat, torch.zeros_like(is_pos), torch.ones_like(is_neg)) # print('ap', dist_ap) # print('an', dist_an) # print('nn', dist_nn) y = dist_an.new().resize_as_(dist_an).fill_(1) if alpha1 > 0: loss_a1 = F.margin_ranking_loss(dist_an, dist_ap, y, margin=alpha1) else: loss_a1 = F.soft_margin_loss(dist_an - dist_ap, y) # fmt: off if loss_a1 == float('Inf'): loss_a1 = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) # fmt: on if alpha2 > 0: loss_a2 = F.margin_ranking_loss(dist_nn, dist_ap, y, margin=alpha2) else: loss_a2 = F.soft_margin_loss(dist_nn - dist_ap, y) # fmt: off if loss_a2 == float('Inf'): loss_a2 = F.margin_ranking_loss(dist_nn, dist_ap, y, margin=0.3) # fmt: on return { 'loss_a1': loss_a1, 'loss_a2': loss_a2 }
def select_mask_logistic_loss(p_m, mask, weight, o_sz=63, g_sz=127): """ 计算图像分割分支的损失函数及精度信息 :param p_m:预测的分割结果 :param mask: 掩膜真实结果 :param weight: :param o_sz:模板的大小 :param g_sz:图像的大小 :return: """ weight = weight.view(-1) pos = Variable(weight.data.eq(1).nonzero().squeeze()) if pos.nelement() == 0: return p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0 # 维度转换 p_m = p_m.permute(0, 2, 3, 1).contiguous().view(-1, 1, o_sz, o_sz) p_m = torch.index_select(p_m, 0, pos) # 2d升采样 p_m = nn.UpsamplingBilinear2d(size=[g_sz, g_sz])(p_m) p_m = p_m.view(-1, g_sz * g_sz) # 对掩膜的真实结果进行处理 mask_uf = F.unfold(mask, (g_sz, g_sz), padding=32, stride=8) mask_uf = torch.transpose(mask_uf, 1, 2).contiguous().view(-1, g_sz * g_sz) mask_uf = torch.index_select(mask_uf, 0, pos) # 计算损失函数 loss = F.soft_margin_loss(p_m, mask_uf) # 计算精度 iou_m, iou_5, iou_7 = iou_measure(p_m, mask_uf) # 返回结果 return loss, iou_m, iou_5, iou_7
def __call__(self, _, global_features, targets): if self._normalize_feature: global_features = normalize(global_features, axis=-1) if self._use_cosine_dist: dist_mat = cosine_dist(global_features, global_features) else: dist_mat = euclidean_dist(global_features, global_features) N = dist_mat.size(0) is_pos = targets.expand(N, N).eq(targets.expand(N, N).t()) is_neg = targets.expand(N, N).ne(targets.expand(N, N).t()) if self._hard_mining: dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg) else: dist_ap, dist_an = weighted_example_mining(dist_mat, is_pos, is_neg) y = dist_an.new().resize_as_(dist_an).fill_(1) if self._margin > 0: loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=self._margin) else: loss = F.soft_margin_loss(dist_an - dist_ap, y) if loss == float('Inf'): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) return { "loss_triplet": loss * self._scale, }
def soft_margin_loss(pos, neg=None, reduction="none"): """Compute the BPR loss.""" if neg is None: x = pos else: x = pos - neg return F.soft_margin_loss(x, torch.ones_like(x), reduction=reduction)
def select_mask_logistic_loss(p_m, mask, weight, o_sz=63, g_sz=127): weight = weight.view(-1) # weight的size为w*h,每个RoW位置的权重由该位置的最大的类别标签决定 pos = Variable(weight.data.eq(1).nonzero().squeeze()) # 挑选出正类来 if pos.nelement() == 0: return p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0 # 下面4行是产生m_n的 p_m = p_m.permute(0, 2, 3, 1).contiguous().view(-1, 1, o_sz, o_sz) # b,w,h,c p_m = torch.index_select(p_m, 0, pos) p_m = nn.UpsamplingBilinear2d(size=[g_sz, g_sz])(p_m) # 双线性插值,以减少计算量 p_m = p_m.view(-1, g_sz * g_sz) # 之所以这么干,是为了计算损失函数L_mask方便 # 下面两行产生cn # Extracts sliding local blocks from a batched input tensor. 可以当官网查看该函数的帮助文档 mask_uf = F.unfold(mask, (g_sz, g_sz), padding=32, stride=8) # 为什么padding要32??? 强行解释可以查看table 8,有注释。 mask_uf = torch.transpose(mask_uf, 1, 2).contiguous().view( -1, g_sz * g_sz) # 4,127*127,25*25 # b w c h # 紧贴下面一行是产生positive cn的 mask_uf = torch.index_select(mask_uf, 0, pos) loss = F.soft_margin_loss(p_m, mask_uf) # 论文中的公式(3),计算L_mask损失函数;只计算正类 iou_m, iou_5, iou_7 = iou_measure(p_m, mask_uf) return loss, iou_m, iou_5, iou_7 # iou_m表示iou,而iou_5, iou_7表示相应的准确率
def forward(self): a = torch.randn(3, 2) b = torch.rand(3, 2) c = torch.rand(3) log_probs = torch.randn(50, 16, 20).log_softmax(2).detach() targets = torch.randint(1, 20, (16, 30), dtype=torch.long) input_lengths = torch.full((16, ), 50, dtype=torch.long) target_lengths = torch.randint(10, 30, (16, ), dtype=torch.long) return len( F.binary_cross_entropy(torch.sigmoid(a), b), F.binary_cross_entropy_with_logits(torch.sigmoid(a), b), F.poisson_nll_loss(a, b), F.cosine_embedding_loss(a, b, c), F.cross_entropy(a, b), F.ctc_loss(log_probs, targets, input_lengths, target_lengths), # F.gaussian_nll_loss(a, b, torch.ones(5, 1)), # ENTER is not supported in mobile module F.hinge_embedding_loss(a, b), F.kl_div(a, b), F.l1_loss(a, b), F.mse_loss(a, b), F.margin_ranking_loss(c, c, c), F.multilabel_margin_loss(self.x, self.y), F.multilabel_soft_margin_loss(self.x, self.y), F.multi_margin_loss(self.x, torch.tensor([3])), F.nll_loss(a, torch.tensor([1, 0, 1])), F.huber_loss(a, b), F.smooth_l1_loss(a, b), F.soft_margin_loss(a, b), F.triplet_margin_loss(a, b, -b), # F.triplet_margin_with_distance_loss(a, b, -b), # can't take variable number of arguments )
def __call__(self, embedding, targets): if self._normalize_feature: embedding = normalize(embedding, axis=-1) all_embedding = embedding all_targets = targets dist_mat = euclidean_dist(embedding, all_embedding) N, M = dist_mat.size() is_pos = targets.view(N, 1).expand(N, M).eq( all_targets.view(M, 1).expand(M, N).t()) is_neg = targets.view(N, 1).expand(N, M).ne( all_targets.view(M, 1).expand(M, N).t()) if self._hard_mining: dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg) else: dist_ap, dist_an = weighted_example_mining(dist_mat, is_pos, is_neg) y = dist_an.new().resize_as_(dist_an).fill_(1) if self._margin > 0: loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=self._margin) else: loss = F.soft_margin_loss(dist_an - dist_ap, y) if loss == float('Inf'): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) return loss * self._scale
def select_mask_logistic_loss(p_m, mask, weight, o_sz=63, g_sz=127, padding=32): weight = weight.view(-1) pos = Variable(weight.data.eq(1).nonzero().squeeze()) if pos.nelement() == 0: return p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0 # print("p_m1:", p_m.shape) # p_m1: torch.Size([16, 3969, 25, 25]) if len(p_m.shape) == 4: # print("p_m1:", p_m.shape) # p_m1: torch.Size([16, 3969, 25, 25]) p_m = p_m.permute(0, 2, 3, 1).contiguous().view(-1, 1, o_sz, o_sz) p_m = torch.index_select(p_m, 0, pos) p_m = torch.nn.UpsamplingBilinear2d(size=[g_sz, g_sz])(p_m) p_m = p_m.view(-1, g_sz * g_sz) # print("p_m:", p_m.shape) # p_m: torch.Size([47, 16129]) else: # print("p_m1:", p_m.shape) # p_m1: torch.Size([576, 16129]) p_m1: torch.Size([9, 16129]) p_m = torch.index_select(p_m, 0, pos) # print("p_m:", p_m.shape) # p_m: torch.Size([339, 16129]) mask_uf = F.unfold(mask, (g_sz, g_sz), padding=padding, stride=8) # print("mask_uf:", mask_uf.shape) # mask_uf: torch.Size([16, 16129, 625]) mask_uf = torch.transpose(mask_uf, 1, 2).contiguous().view(-1, g_sz * g_sz) # print("mask_uf:", mask_uf.shape) # mask_uf: torch.Size([10000, 16129]) mask_uf = torch.index_select(mask_uf, 0, pos) # print("mask_uf:", mask_uf.shape) # mask_uf: torch.Size([47, 16129]) loss = F.soft_margin_loss(p_m, mask_uf) iou_m, iou_5, iou_7 = iou_measure(p_m, mask_uf) # print("ioum:", iou_m) return loss, iou_m, iou_5, iou_7
def do_generate(self, paths, step, optimizer, dataset, test_index, model_, pooling_model, deterministic=False, use_half=False, verbose=False): k = step // 1000 index = 1 trn_loader = DataLoader( dataset, collate_fn=lambda batch: env.collate_unknownspeaker_samples(batch), batch_size=4, num_workers=4, shuffle=True, pin_memory=True) iters = len(trn_loader) loss_val = 0. epoch = 20 for e in range(epoch): for i, (wav, query_mfcc, label) in enumerate(trn_loader): model_.train() pooling_model.train() wav = wav.cuda() query_mfcc16 = query_mfcc.cuda() label = label.float().cuda() os.makedirs(paths.gen_path(), exist_ok=True) #model_ = QueryEncoder() query_encoded = model_(query_mfcc16) #[0:1000,:] out = self.forward_QbE(wav, query_encoded[:, -1, :], pooling_model, verbose=verbose, use_half=use_half, index=index) # loss = F.binary_cross_entropy(out, label) loss = F.soft_margin_loss(out, label) loss_val += loss.item() if i % 20 == 1: print(" VQVAE: Loss: ", loss.item(), " at iteration ", i, " labels are ", label) optimizer.zero_grad() loss.backward() optimizer.step() # batchsize index += 4
def __call__(self, query_embedding, occluded_embedding, targets, occluded_targets): query_embedding = query_embedding.squeeze() occluded_embedding = occluded_embedding.squeeze() if self._normalize_feature: query_embedding = normalize(query_embedding, axis=-1) occluded_embedding = normalize(occluded_embedding, axis=-1) query_dist_mat = batch_drop_euclidean_dist(query_embedding, query_embedding) occluded_dist_mat = euclidean_dist(occluded_embedding, occluded_embedding) N, M = query_dist_mat.size() is_pos = targets.view(N, 1).expand(N, M).eq( targets.view(M, 1).expand(M, N).t()) is_neg = ~is_pos same_occluded = occluded_targets.view(N, 1).expand(N, M).eq( occluded_targets.view(M, 1).expand(M, N).t()) diff_occluded = ~same_occluded if self._hard_mining: query_dist_ap, query_dist_an, occluded_dist_ap, occluded_dist_an = \ new_hard_example_mining(query_dist_mat, occluded_dist_mat, is_pos*diff_occluded, is_neg*same_occluded) # else: # dist_ap, dist_an = weighted_example_mining(query_dist_mat, is_pos*diff_occluded, is_neg*same_occluded) query_y = query_dist_an.new().resize_as_(query_dist_an).fill_(1) occluded_y = occluded_dist_an.new().resize_as_(occluded_dist_an).fill_( 1) if self._margin > 0: loss = F.margin_ranking_loss(query_dist_an, query_dist_ap, query_y, margin=self._margin)\ + self._lambda * F.margin_ranking_loss(occluded_dist_an, occluded_dist_ap, occluded_y, margin=self._margin) else: loss = F.soft_margin_loss(query_dist_an - query_dist_ap, query_y)\ + self._lambda * F.soft_margin_loss(occluded_dist_an - occluded_dist_ap, occluded_y) if loss == float('Inf'): loss = F.margin_ranking_loss(query_dist_an, query_dist_ap, query_y, margin=0.3) \ + self._lambda * F.margin_ranking_loss(occluded_dist_an, occluded_dist_ap, occluded_y, margin=0.3) return loss * self._scale
def triplet_loss(express_vec, labels, weight=1.0): _, labels = torch.max(labels, 1) anchor, positives, negatives = get_triplets(express_vec, labels) num_samples = anchor.shape[0] y = torch.ones((num_samples, 1)).view(-1) if anchor.is_cuda: y = y.cuda() ap_dist = torch.norm(anchor - positives, 2, dim=1).view(-1) an_dist = torch.norm(anchor - negatives, 2, dim=1).view(-1) cost = F.soft_margin_loss(an_dist - ap_dist, y) cost *= weight return cost
def forward(self, embedding: torch.Tensor, targets: torch.Tensor) -> torch.Tensor: """Forward Method. Args: embedding: The output of the network. targets: The targets. Returns: The computed Triplet Loss. """ distance_matrix = (cosine_dist(embedding, embedding) if self.normalize_features else euclidean_dist( embedding, embedding)) n = distance_matrix.size(0) pos_idxs = targets.view(n, 1).expand(n, n).eq( targets.view(n, 1).expand(n, n).t()).float() neg_idxs = targets.view(n, 1).expand(n, n).ne( targets.view(n, 1).expand(n, n).t()).float() if self.hard_mining: dist_ap, dist_an = hard_example_mining( distance_matrix=distance_matrix, pos_idxs=pos_idxs, neg_idxs=neg_idxs) else: dist_ap, dist_an = weighted_example_mining( distance_matrix=distance_matrix, pos_idxs=pos_idxs, neg_idxs=neg_idxs) y = dist_an.new().resize_as_(dist_an).fill_(1) if self.margin is not None and self.margin > 0: loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=self.margin) else: loss = F.soft_margin_loss(dist_an - dist_ap, y) # fmt: off if loss == float("Inf"): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) # fmt: on return loss
def forward(self, pred, target): """ Arguments: pred(Tensor[N, C]): Predicted scores of N samples for C classes target(Tensor[N, C]): Binary labels """ # construct positive/negative pairs in cpu due to high memory consumption target_cpu = target.clone().cpu() # find the indices of all positive and negatie logits all_p = target.nonzero() all_n = (1 - target).nonzero() # pair up positive and negative logits regardless of class all_p_x, all_n_x = torch.meshgrid(all_p[:, 0], all_n[:, 0]) all_p_y, all_n_y = torch.meshgrid(all_p[:, 1], all_n[:, 1]) # only keep pairs from the same class mask = torch.eq(all_p_y, all_n_y) p_x = all_p_x[mask].to(pred.device) p_y = all_p_y[mask].to(pred.device) n_x = all_n_x[mask].to(pred.device) n_y = all_n_y[mask].to(pred.device) # randomly sample a proportion of pairs if self._sampling_ratio != 1: sample_inds = torch.randperm( p_x.numel())[:int(p_x.numel() * self._sampling_ratio)] p_x = p_x[sample_inds] p_y = p_x[sample_inds] n_x = p_x[sample_inds] n_y = p_x[sample_inds] loss = F.soft_margin_loss(pred[p_x, p_y] - pred[n_x, n_y], torch.ones_like(p_x, dtype=torch.float32, device=pred.device), reduction='none') if self._reduction == 'none': return loss elif self._reduction == 'mean': return torch.mean(loss) elif self._reduction == 'sum': return torch.sum(loss) else: raise NotImplementedError('Unsupported reduction mode {}'.format( self._reduction))
def test_soft_margin_loss(self): inp = torch.randn(32, 128, device='cuda', dtype=self.dtype, requires_grad=True) target = torch.randn(32, 128, device='cuda', dtype=self.dtype, requires_grad=False) output = F.soft_margin_loss(inp, target, size_average=None, reduce=None, reduction='mean')
def select_mask_logistic_loss(p_m, mask, weight, o_sz=63, g_sz=127): weight = weight.view(-1) pos = Variable(weight.data.eq(1).nonzero().squeeze()) if pos.nelement() == 0: return p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0, p_m.sum() * 0 p_m = p_m.permute(0, 2, 3, 1).contiguous().view(-1, 1, o_sz, o_sz) p_m = torch.index_select(p_m, 0, pos) p_m = nn.UpsamplingBilinear2d(size=[g_sz, g_sz])(p_m) p_m = p_m.view(-1, g_sz * g_sz) mask_uf = F.unfold(mask, (g_sz, g_sz), padding=32, stride=8) mask_uf = torch.transpose(mask_uf, 1, 2).contiguous().view(-1, g_sz * g_sz) mask_uf = torch.index_select(mask_uf, 0, pos) loss = F.soft_margin_loss(p_m, mask_uf) iou_m, iou_5, iou_7 = iou_measure(p_m, mask_uf) return loss, iou_m, iou_5, iou_7
def _triplet_loss(self, feat_q, targets): dist_mat = euclidean_dist(feat_q, self.queue.t()) N, M = dist_mat.size() is_pos = targets.view(N, 1).expand(N, M).eq(self.queue_label.expand(N, M)).float() sorted_mat_distance, positive_indices = torch.sort(dist_mat + (-9999999.) * (1 - is_pos), dim=1, descending=True) dist_ap = sorted_mat_distance[:, 0] sorted_mat_distance, negative_indices = torch.sort(dist_mat + 9999999. * is_pos, dim=1, descending=False) dist_an = sorted_mat_distance[:, 0] y = dist_an.new().resize_as_(dist_an).fill_(1) loss = F.soft_margin_loss(dist_an - dist_ap, y) if loss == float('Inf'): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) return loss
def triplet_loss(embedding, targets, margin, norm_feat, hard_mining): """Modified from Tong Xiao's open-reid (https://github.com/Cysu/open-reid). Related Triplet Loss theory can be found in paper 'In Defense of the Triplet Loss for Person Re-Identification'.""" # import pdb; pdb.set_trace() if norm_feat: embedding = normalize(embedding, axis=-1) # For distributed training, gather all features from different process. if comm.get_world_size() > 1: all_embedding = torch.cat(GatherLayer.apply(embedding), dim=0) all_targets = concat_all_gather(targets) else: all_embedding = embedding all_targets = targets dist_mat = euclidean_dist(all_embedding, all_embedding) #dist_mat = cosine_dist(all_embedding, all_embedding) N, N = dist_mat.size() is_pos = all_targets.view(N, 1).expand(N, N).eq( all_targets.view(N, 1).expand(N, N).t()) is_neg = all_targets.view(N, 1).expand(N, N).ne( all_targets.view(N, 1).expand(N, N).t()) if hard_mining: dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg) else: dist_ap, dist_an = weighted_example_mining(dist_mat, is_pos, is_neg) y = dist_an.new().resize_as_(dist_an).fill_(1) if margin > 0: loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=margin) else: loss = F.soft_margin_loss(dist_an - dist_ap, y) # fmt: off if loss == float('Inf'): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) # fmt: on return loss
def loss_per_level(self, estDisp, gtDisp, label): N, C, H, W = estDisp.shape scaled_gtDisp = gtDisp scale = 1.0 if gtDisp.shape[-2] != H or gtDisp.shape[-1] != W: # compute scale per level and scale gtDisp scale = gtDisp.shape[-1] / (W * 1.0) scaled_gtDisp = gtDisp / scale scaled_gtDisp = self.scale_func(scaled_gtDisp, (H, W)) # mask for valid disparity # (start disparity, max disparity / scale) # Attention: the invalid disparity of KITTI is set as 0, be sure to mask it out mask = (scaled_gtDisp > self.start_disp) & (scaled_gtDisp < (self.max_disp / scale)) if mask.sum() < 1.0: print( 'Relative loss: there is no point\'s disparity is in ({},{})!'. format(self.start_disp, self.max_disp / scale)) loss = (torch.abs(estDisp - scaled_gtDisp) * mask.float()).mean() return loss # relative loss valid_pixel_number = mask.float().sum() diff = scaled_gtDisp[mask] - estDisp[mask] label = label[mask] # some value which is over large for torch.exp() is not suitable for soft margin loss # get absolute value great than 66 over_large_mask = torch.gt(torch.abs(diff), 66) over_large_diff = diff[over_large_mask] # get absolute value smaller than 66 proper_mask = torch.le(torch.abs(diff), 66) proper_diff = diff[proper_mask] # generate lable for soft margin loss label = label[proper_mask] loss = F.soft_margin_loss( proper_diff, label, reduction='sum') + torch.abs(over_large_diff).sum() loss = loss / valid_pixel_number return loss
def __call__(self, embedding, targets): if self._normalize_feature: embedding = normalize(embedding, axis=-1) # For distributed training, gather all features from different process. if comm.get_world_size() > 1: all_embedding = concat_all_gather(embedding) all_targets = concat_all_gather(targets) else: all_embedding = embedding all_targets = targets dist_mat = euclidean_dist(all_embedding, all_embedding) N = dist_mat.size(0) is_pos = all_targets.view(N, 1).expand(N, N).eq( all_targets.view(N, 1).expand(N, N).t()) is_neg = all_targets.view(N, 1).expand(N, N).ne( all_targets.view(N, 1).expand(N, N).t()) if self._hard_mining: dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg) else: dist_ap, dist_an = weighted_example_mining(dist_mat, is_pos, is_neg) y = dist_an.new().resize_as_(dist_an).fill_(1) if self._margin > 0: loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=self._margin) else: loss = F.soft_margin_loss(dist_an - dist_ap, y) if loss == float('Inf'): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) return loss * self._scale
def configure_criterion(self, y, t): criterion = F.cross_entropy(y, t) if self.hparams.criterion == "cross_entropy": criterion = F.cross_entropy(y, t) elif self.hparams.criterion == "binary_cross_entropy": criterion = F.binary_cross_entropy(y, t) elif self.hparams.criterion == "binary_cross_entropy_with_logits": criterion = F.binary_cross_entropy_with_logits(y, t) elif self.hparams.criterion == "poisson_nll_loss": criterion = F.poisson_nll_loss(y, t) elif self.hparams.criterion == "hinge_embedding_loss": criterion = F.hinge_embedding_loss(y, t) elif self.hparams.criterion == "kl_div": criterion = F.kl_div(y, t) elif self.hparams.criterion == "l1_loss": criterion = F.l1_loss(y, t) elif self.hparams.criterion == "mse_loss": criterion = F.mse_loss(y, t) elif self.hparams.criterion == "margin_ranking_loss": criterion = F.margin_ranking_loss(y, t) elif self.hparams.criterion == "multilabel_margin_loss": criterion = F.multilabel_margin_loss(y, t) elif self.hparams.criterion == "multilabel_soft_margin_loss": criterion = F.multilabel_soft_margin_loss(y, t) elif self.hparams.criterion == "multi_margin_loss": criterion = F.multi_margin_loss(y, t) elif self.hparams.criterion == "nll_loss": criterion = F.nll_loss(y, t) elif self.hparams.criterion == "smooth_l1_loss": criterion = F.smooth_l1_loss(y, t) elif self.hparams.criterion == "soft_margin_loss": criterion = F.soft_margin_loss(y, t) return criterion
def triplet_loss(embedding, targets, margin: float, norm_feat=False, mining='weightedexample', reduction='mean'): """Modified from Tong Xiao's open-reid (https://github.com/Cysu/open-reid). Related Triplet Loss theory can be found in paper 'In Defense of the Triplet Loss for Person Re-Identification'. Args: embedding (Tensor[float]): [N, D] targets (Tensor[long]): [N] margin (float): [description] norm_feat (bool, optional): whether the feature is normalized. if so, use cos distance. else euclidean distance. mining (str, optional): Defaults to 'weightedexample'. reduction (str, optional): Defaults to 'mean'. Raises: ValueError: if mining is not matched Returns: Tensor: [description] """ if norm_feat: dist_mat = cosine_dist(embedding, embedding) else: dist_mat = euclidean_dist(embedding, embedding) # For distributed training, gather all features from different process. # if comm.get_world_size() > 1: # all_embedding = torch.cat(GatherLayer.apply(embedding), dim=0) # all_targets = concat_all_gather(targets) # else: # all_embedding = embedding # all_targets = targets N = dist_mat.size(0) is_pos = (targets.view(N, 1).expand(N, N).eq( targets.view(N, 1).expand(N, N).t()).float()) is_neg = (targets.view(N, 1).expand(N, N).ne( targets.view(N, 1).expand(N, N).t()).float()) if mining == 'hardmining': dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg) elif mining == 'weightedexample': dist_ap, dist_an = weighted_example_mining(dist_mat, is_pos, is_neg) else: raise ValueError(mining) y = dist_an.new().resize_as_(dist_an).fill_(1) if margin > 0: loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=margin, reduction='none') else: loss = F.soft_margin_loss(dist_an - dist_ap, y, reduction='none') if torch.any(torch.isinf(loss)): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3, reduction='none') return _reduct(loss, reduction) # loss: [N]
def __call__(self, output: torch.FloatTensor, labels: torch.FloatTensor) -> torch.FloatTensor: return functional.soft_margin_loss(output, labels)
def soft_margin(y_pred, y_true): return F.soft_margin_loss(y_pred, y_true)
continue elif not list_is_small: query_embs, para_embs, token_probs, is_caps_and_quotes, columns, tfs, idfs= prepare_query2(queries, current_index, token_stats, device, bc, bc_para, all_paras, paragraph_sample_index) if j == 0: sample = (queries, is_positive, paragraph_sample_index, query_embs, para_embs, token_probs, is_caps_and_quotes, columns, tfs, idfs) list_of_samples.append(sample) paragraph_sample_index = get_random_indices(len(columns))[0] ##solr_weights, out, out_pre = h_model(True, query_embs, para_embs, token_probs, is_caps_and_quotes, columns, device, (average_out-.5), tfs, idfs) solr_weights, out, out_pre = h_model(query_ids, paragraph_ids, device, tfs, idfs, para_mask=para_mask, query_mask=query_mask, is_train=is_train) #loss = cal_loss(out, target, True) #print(target.size()) #print(out) target = 1. if is_positive == 1 else 0. target = torch.tensor([target], device=device) loss = F.soft_margin_loss(out, target) #loss = F.cross_entropy(out, target, reduction='sum') loss.backward() #print(loss) optimizer.step() optimizer.zero_grad() if i%1000==1 and not data_is_small: #torch.save(h_model.state_dict(), str("model_just_para_and_tokens_v" + str(step) + ".pth")) torch.save(model.state_dict(), output_model_file) model.config.to_json_file(output_config_file) tokenizer.save_vocabulary(output_vocab_file) if (step % 100 == 0 and len(queries[current_index].tokens) > 1) or data_is_small: total_correct, total = print_statistics2(step, loss, total_correct, total, is_train, solr_weights.cpu().detach(), queries[current_index], out, target) if total_correct/total > .5: pass else:
def soft_margin_loss(x): target = torch.ones_like(x) return F.soft_margin_loss(x, target, reduction="none")
def triplet_loss(embedding, targets, margin, norm_feat, hard_mining, dist_type, loss_type, domain_labels=None, pos_flag=[1, 0, 0], neg_flag=[0, 0, 1]): r"""Modified from Tong Xiao's open-reid (https://github.com/Cysu/open-reid). Related Triplet Loss theory can be found in paper 'In Defense of the Triplet Loss for Person Re-Identification'.""" if norm_feat: embedding = normalize(embedding, axis=-1) # For distributed training, gather all features from different process. if comm.get_world_size() > 1: all_embedding = concat_all_gather(embedding) all_targets = concat_all_gather(targets) else: all_embedding = embedding all_targets = targets if dist_type == 'euclidean': dist_mat = euclidean_dist(all_embedding, all_embedding) elif dist_type == 'cosine': dist_mat = cosine_dist(all_embedding, all_embedding) N = dist_mat.size(0) if (pos_flag == [1, 0, 0] and neg_flag == [0, 1, 1]) or domain_labels == None: is_pos = all_targets.view(N, 1).expand(N, N).eq( all_targets.view(N, 1).expand(N, N).t()) is_neg = all_targets.view(N, 1).expand(N, N).ne( all_targets.view(N, 1).expand(N, N).t()) else: vec1 = copy.deepcopy(all_targets) for i in range(N): vec1[i] = i # [0,1,2,3,4,~~] is_same_img = vec1.expand(N, N).eq(vec1.expand(N, N).t()) is_same_instance = all_targets.view(N, 1).expand(N, N).eq( all_targets.view(N, 1).expand(N, N).t()) is_same_domain = domain_labels.view(N, 1).expand(N, N).eq( domain_labels.view(N, 1).expand(N, N).t()) set0 = is_same_img set_all = [] set_all.extend([is_same_instance * (is_same_img == False)]) set_all.extend([(is_same_instance == False) * (is_same_domain == True) ]) set_all.extend([is_same_domain == False]) is_pos = copy.deepcopy(set0) is_neg = copy.deepcopy(set0 == False) is_neg[:] = False for i, bool_flag in enumerate(pos_flag): if bool_flag == 1: is_pos += set_all[i] for i, bool_flag in enumerate(neg_flag): if bool_flag == 1: is_neg += set_all[i] # print(pos_flag) # print(is_pos.type(torch.IntTensor)) # print(neg_flag) # print(is_neg.type(torch.IntTensor)) if hard_mining: dist_ap, dist_an = hard_example_mining(dist_mat, is_pos, is_neg) else: dist_ap, dist_an = weighted_example_mining(dist_mat, is_pos, is_neg) y = dist_an.new().resize_as_(dist_an).fill_(1) if margin > 0: # all(sum(is_pos) == 1) loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=margin) else: if loss_type == 'logistic': loss = F.soft_margin_loss(dist_an - dist_ap, y) # fmt: off if loss == float('Inf'): loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=0.3) # fmt: on elif loss_type == 'hinge': loss = F.margin_ranking_loss(dist_an, dist_ap, y, margin=margin) return loss
def soft_margin_loss(input, target, *args, **kwargs): return F.soft_margin_loss(input.F, target, *args, **kwargs)
def rp_loss(model, x, y): out = model(x) return soft_margin_loss(out, y)