def forward(self, outs, targets, imgs_ts=None): ''' 模型尺寸list[20,10,5,3] :param outs: torch.Size([2, 534, 7]) in 160 输出[] :param targets: 'image_id': 413, 'size': tensor([500., 309.]) 'boxes': tensor([[0.31400, 0.31715, 0.71000, 0.60841]]), 'labels': tensor([1.]) :param imgs_ts: :return: ''' cfg = self.cfg device = outs.device batch, dim_total, pdim = outs.shape # 1 + cfg.NUM_CLASSES + 1 + 4 + cfg.NUM_KEYPOINTS * 2 # back cls centerness ltrb positivesample iou area gdim = 1 + cfg.NUM_CLASSES + 1 + 4 + 1 + 1 + 1 gres = torch.empty((batch, dim_total, gdim), device=device) for i in range(batch): gboxes_ltrb_b = targets[i]['boxes'] glabels_b = targets[i]['labels'] gres[i] = match4fcos_v2( gboxes_ltrb_b=gboxes_ltrb_b, glabels_b=glabels_b, gdim=gdim, pcos=outs, img_ts=imgs_ts[i], cfg=cfg, ) s_ = 1 + cfg.NUM_CLASSES # outs = outs[:, :, :s_ + 1].sigmoid() mask_pos = gres[:, :, 0] == 0 # 背景为0 是正例 nums_pos = torch.sum(mask_pos, dim=-1) nums_pos = torch.max(nums_pos, torch.ones_like(nums_pos, device=device)) # back cls centerness ltrb positivesample iou(这个暂时无用) area [2125, 12] ''' ---------------- cls损失 计算全部样本,正反例,正例为框内本例---------------- ''' # obj_cls_loss = BCE_focal_loss() # 这里多一个背景一起算 pcls_sigmoid = outs[:, :, :s_].sigmoid() gcls = gres[:, :, :s_] # l_cls = torch.mean(obj_cls_loss(pcls_sigmoid, gcls) / nums_pos) l_cls_pos, l_cls_neg = focalloss_fcos(pcls_sigmoid, gcls) l_cls_pos = torch.mean( torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos) l_cls_neg = torch.mean( torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos) ''' ---------------- conf损失 只计算半径正例 center_ness---------------- ''' # 和 positive sample 算正例 mask_pp = gres[:, :, s_ + 1 + 4] == 1 pconf_sigmoid = outs[:, :, s_].sigmoid() # center_ness gcenterness = gres[:, :, s_] # (nn,1) # 使用centerness # _loss_val = x_bce(pconf_sigmoid, gcenterness, reduction="none") _loss_val = x_bce(pconf_sigmoid, torch.ones_like(pconf_sigmoid), reduction="none") # 用半径1 # 只算半径正例,提高准确性 l_conf = 5. * torch.mean( torch.sum(_loss_val * mask_pp.float(), dim=-1) / nums_pos) ''' ---------------- box损失 计算框内正例---------------- ''' # conf1 + cls3 + reg4 poff_ltrb = outs[:, :, s_:s_ + 4] # 这个全是特图的距离 全rule 或 exp # goff_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4] g_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4] # 这里是解析归一化图 归一化与特图计算的IOU是一致的 pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb) p_ltrb_pos = pboxes_ltrb[mask_pos] g_ltrb_pos = g_ltrb[mask_pos] iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True) # 使用 iou 与 1 进行bce debug iou.isnan().any() or iou.isinf().any() l_reg = 5 * torch.mean((1 - iou) * gcenterness[mask_pos]) l_total = l_cls_pos + l_cls_neg + l_conf + l_reg log_dict = {} log_dict['l_total'] = l_total.item() log_dict['l_cls_pos'] = l_cls_pos.item() log_dict['l_cls_neg'] = l_cls_neg.item() log_dict['l_conf'] = l_conf.item() log_dict['l_reg'] = l_reg.item() # log_dict['l_iou_max'] = iou.max().item() return l_total, log_dict
def forward(self, pyolos, targets, imgs_ts=None): ''' :param pyolos: torch.Size([2, 45, 13, 13]) :param targets: :param imgs_ts: :return: ''' cfg = self.cfg device = pyolos.device batch, c, h, w = pyolos.shape # torch.Size([2, 45, 13, 13]) # [3, 40, 13, 13] -> [3, 8, 5, 13*13] -> [3, 169, 5, 8] pyolos = pyolos.view(batch, 1 + cfg.NUM_CLASSES + 4, cfg.NUM_ANC, - 1).permute(0, 3, 2, 1).contiguous() # [3, 169, 5, 8] -> [3, 169*5, 8] pyolos = pyolos.view(batch, h * w * cfg.NUM_ANC, -1) # pyolos = pyolos.view(batch, -1, s_ + 4) preg = pyolos[..., 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 4] # torch.Size([2, 169, 5, 4]) pltrb = boxes_decode4yolo2_v2(preg, h, w, cfg) # 输出原图归一化 用于更新conf [2, 845, 4] '''--------------gt匹配---------------''' # conf-1, cls-1, txywh-4, weight-1, gltrb-4 if cfg.MODE_TRAIN == 99 or cfg.MODE_TRAIN == 98: gdim = 1 + 1 + 4 + 1 + 4 gyolos = torch.empty((batch, h, w, cfg.NUM_ANC, gdim), device=device) # 匹配GT for i, target in enumerate(targets): # batch遍历 gboxes_ltrb_b = target['boxes'] # ltrb glabels_b = target['labels'] # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 if cfg.MODE_TRAIN == 99 or cfg.MODE_TRAIN == 98: gyolos[i] = fmatch4yolov2_99( gboxes_ltrb_b=gboxes_ltrb_b, glabels_b=glabels_b, grid=h, # 7 只有一层 gdim=gdim, device=device, cfg=cfg, preg_b=preg[i], img_ts=imgs_ts[i], ) '''可视化验证''' if cfg.IS_VISUAL: # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 gyolo_test = gyolos[i].clone() # torch.Size([32, 13, 13, 9]) gyolo_test = gyolo_test.view(-1, gdim) gconf_one = gyolo_test[:, 0] # mask_pos = torch.logical_or(gconf_one == 1, gconf_one == -1) mask_pos_2d = gconf_one == 1 gtxywh = gyolo_test[:, 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 4] # 这里是修复是 xy _xy_grid = gtxywh[:, :2] + f_mershgrid(h, w, is_rowcol=False, num_repeat=cfg.NUM_ANC).to(device) hw_ts = torch.tensor((h, w), device=device) gtxywh[:, :2] = torch.true_divide(_xy_grid, hw_ts) gtxywh = gtxywh[mask_pos_2d] gtxywh[:, 2:4] = torch.exp(gtxywh[:, 2:]) / h # 原图归一化 from f_tools.pic.enhance.f_data_pretreatment4pil import f_recover_normalization4ts img_ts = f_recover_normalization4ts(imgs_ts[i]) from torchvision.transforms import functional as transformsF img_pil = transformsF.to_pil_image(img_ts).convert('RGB') import numpy as np img_np = np.array(img_pil) f_show_od_np4plt(img_np, gboxes_ltrb=gboxes_ltrb_b.cpu() , pboxes_ltrb=xywh2ltrb(gtxywh.cpu()), is_recover_size=True, grids=(h, w)) # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 # torch.Size([32, 13, 13, 5, 11]) -> [32, 13*13*5, 11] ->[2, 845, 11] gyolos = gyolos.view(batch, -1, gdim) gconf = gyolos[:, :, 0] # 正例使用1 torch.Size([32, 910]) # mask_pos_3d = gyolos[:, :, :1] > 0 # mask_neg_3d = gyolos[:, :, :1] == 0 mask_pos_2d = gconf > 0 mask_neg_2d = gconf == 0 # 忽略-1 不管 nums_pos = (mask_pos_2d.sum(-1).to(torch.float)).clamp(min=torch.finfo(torch.float16).eps) nums_neg = (mask_neg_2d.sum(-1).to(torch.float)).clamp(min=torch.finfo(torch.float16).eps) pyolos_pos = pyolos[mask_pos_2d] # torch.Size([32, 845, 8]) -> torch.Size([40, 8]) gyolos_pos = gyolos[mask_pos_2d] # torch.Size([32, 845, 13]) -> torch.Size([40, 8]) # [2, 845, 4] -> # iou_zg = bbox_iou4one(pltrb, gyolos[..., 1 + 1 + 4 + 1:1 + 1 + 4 + 1 + 4], is_giou=True) iou_zg = bbox_iou4one(pltrb, gyolos[..., 1 + 1 + 4 + 1:1 + 1 + 4 + 1 + 4], is_ciou=True) ''' ----------------cls损失---------------- ''' # pcls_sigmoid_pos = pyolos_pos[:, 1:1 + cfg.NUM_CLASSES].sigmoid() pcls_pos = pyolos_pos[:, 1:1 + cfg.NUM_CLASSES] gcls_pos = gyolos_pos[:, 1].long() # torch.Size([3, 4]) ^^ tensor([2., 2., 3.]) _loss_val = F.cross_entropy(pcls_pos, gcls_pos, reduction="none") # _loss_val = x_bce(pcls_sigmoid_pos, gcls_pos, reduction="none") # torch.Size([46, 3]) # torch.Size([46, 3]) -> val l_cls = _loss_val.sum(-1).mean() ''' ----------------conf损失 ---------------- ''' pconf_sigmoid = pyolos[:, :, 0].sigmoid() # 这个需要归一化 torch.Size([3, 845]) # ------------conf-mse ------------ _loss_val = F.mse_loss(pconf_sigmoid, iou_zg, reduction="none") # 这理用的IOU l_conf_pos = ((_loss_val * mask_pos_2d).sum(-1) / nums_pos).mean() * 5. l_conf_neg = ((_loss_val * mask_neg_2d).sum(-1) / nums_neg).mean() * 1. ''' ---------------- box损失 ----------------- ''' pxy_pos_sigmoid = pyolos_pos[:, 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 2].sigmoid() # 这个需要归一化 pwh_pos_scale = pyolos_pos[:, 1 + cfg.NUM_CLASSES + 2:1 + cfg.NUM_CLASSES + 4] weight_pos = gyolos_pos[:, 1 + 1 + 4 + 1] # torch.Size([32, 845]) gtxy_pos = gyolos_pos[:, 1 + 1:1 + 1 + 2] # [nn] gtwh_pos = gyolos_pos[:, 1 + 1 + 2:1 + 1 + 4] _loss_val = x_bce(pxy_pos_sigmoid, gtxy_pos, reduction="none") l_txty = (_loss_val.sum(-1) * weight_pos).mean() _loss_val = F.mse_loss(pwh_pos_scale, gtwh_pos, reduction="none") l_twth = (_loss_val.sum(-1) * weight_pos).mean() ''' ---------------- loss完成 ----------------- ''' log_dict = {} l_total = l_conf_pos + l_conf_neg + l_cls + l_txty + l_twth log_dict['l_total'] = l_total.item() log_dict['l_xy'] = l_txty.item() log_dict['l_wh'] = l_twth.item() log_dict['l_conf_pos'] = l_conf_pos.item() log_dict['l_conf_neg'] = l_conf_neg.item() log_dict['l_cls'] = l_cls.item() return l_total, log_dict
def forward(self, p_center, targets, imgs_ts=None): ''' :param p_center: :param targets: list target['boxes'] = target['boxes'].to(device) target['labels'] = target['labels'].to(device) target['size'] = target['size'] target['image_id'] = int :param imgs_ts: :return: ''' cfg = self.cfg pcls, ptxy, ptwh = p_center device = pcls.device batch, c, h, w = pcls.shape # b,c,h,w -> b,h,w,c -> b,h*w,c pcls = pcls.permute(0, 2, 3, 1).contiguous().view(batch, -1, self.cfg.NUM_CLASSES) ptxy = ptxy.permute(0, 2, 3, 1).contiguous().view(batch, -1, 2) ptwh = ptwh.permute(0, 2, 3, 1).contiguous().view(batch, -1, 2) fsize_wh = torch.tensor([h, w], device=device) # num_class + txywh + weight + gt4 conf通过高斯生成 热力图层数表示类别索引 if cfg.NUM_KEYPOINTS > 0: gdim = cfg.NUM_CLASSES + cfg.NUM_KEYPOINTS * 2 + 4 + 1 + 4 else: gdim = cfg.NUM_CLASSES + 4 + 1 + 4 gres = torch.empty((batch, h, w, gdim), device=device) # 匹配GT for i, target in enumerate(targets): # batch 遍历每一张图 gboxes_ltrb_b = targets[i]['boxes'] glabels_b = targets[i]['labels'] # 处理这张图的所有标签 gres[i] = match4center(gboxes_ltrb_b=gboxes_ltrb_b, glabels_b=glabels_b, fsize_wh=fsize_wh, dim=gdim, cfg=cfg, ) if cfg.IS_VISUAL: from f_tools.pic.enhance.f_data_pretreatment4pil import f_recover_normalization4ts _img_ts = f_recover_normalization4ts(imgs_ts[i].clone()) from torchvision.transforms import functional as transformsF img_pil = transformsF.to_pil_image(_img_ts).convert('RGB') import numpy as np # img_np = np.array(img_pil) '''plt画图部分''' from matplotlib import pyplot as plt plt.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文标签 plt.rcParams['axes.unicode_minus'] = False # 这里的热力图肯定的偏差 [128,128] data_hot = torch.zeros_like(gres[i, :, :, 0]) # 只需要一层即可 for label in glabels_b.unique(): # print(ids2classes[str(int(label))]) # 类别合并输出 flog.debug(' %s', gres[i, :, :, 3:7][gres[i, :, :, (label - 1).long()] == 1]) torch.max(data_hot, gres[i, :, :, (label - 1).long()], out=data_hot) # 这里是类别合并 plt.imshow(data_hot.cpu()) plt.imshow(img_pil.resize(fsize_wh), alpha=0.7) plt.colorbar() # x,y表示横纵坐标,color表示颜色:'r':红 'b':蓝色 等,marker:标记,edgecolors:标记边框色'r'、'g'等,s:size大小 boxes_xywh_cpu = ltrb2xywh(gboxes_ltrb_b).cpu() fsize_cpu = fsize_wh.cpu() xys_f = boxes_xywh_cpu[:, :2] * fsize_cpu plt.scatter(xys_f[:, 0], xys_f[:, 1], color='r', s=5) # 红色 boxes_ltrb_cpu = gboxes_ltrb_b.cpu() boxes_ltrb_f = boxes_ltrb_cpu * fsize_cpu.repeat(2) current_axis = plt.gca() for i, box_ltrb_f in enumerate(boxes_ltrb_f): l, t, r, b = box_ltrb_f # ltwh current_axis.add_patch(plt.Rectangle((l, t), r - l, b - t, color='green', fill=False, linewidth=2)) # current_axis.text(l, t - 2, ids2classes[int(glabels[i])], size=8, color='white', # bbox={'facecolor': 'green', 'alpha': 0.6}) plt.show() gres = gres.reshape(batch, -1, gdim) ''' ---------------- cls损失 只计算正例---------------- ''' gcls = gres[:, :, :cfg.NUM_CLASSES] # mask_pos_3d = gcls > 0 # torch.Size([3, 16384, 3]) # mask_neg_3d = gcls == 0 mask_pos_3d = gcls == 1 # 只有中心点为1正例 torch.Size([3, 16384, 3]) mask_neg_3d = gcls != 1 nums_pos = torch.sum(torch.sum(mask_pos_3d, dim=-1), dim=-1) # mask_pos_2d = torch.any(mask_pos_3d, -1) # focloss pcls_sigmoid = pcls.sigmoid() l_cls_pos, l_cls_neg = focalloss_center(pcls_sigmoid, gcls) l_cls_pos = torch.mean(torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos) l_cls_neg = torch.mean(torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos) # l_cls_neg = l_cls_neg.sum(-1).sum(-1).mean() # 等价 ''' ---------------- box损失 ----------------- ''' log_dict = {} # num_class + txywh + weight + gt4 if cfg.MODE_TRAIN == 2: # iou ptxywh = torch.cat([ptxy, ptwh], dim=-1) pboxes_ltrb = boxes_decode4center(self.cfg, fsize_wh, ptxywh) mask_pos_2d = torch.any(mask_pos_3d, -1) # torch.Size([16, 16384]) # torch.Size([16, 16384, 4]) -> torch.Size([19, 4]) p_ltrb_pos = pboxes_ltrb[mask_pos_2d] g_ltrb_pos = gres[..., cfg.NUM_CLASSES + 4 + 1:cfg.NUM_CLASSES + 4 + 1 + 4][mask_pos_2d] iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True) l_reg = 5 * torch.mean(1 - iou) l_total = l_cls_pos + l_cls_neg + l_reg log_dict['l_total'] = l_total.item() log_dict['l_cls_pos'] = l_cls_pos.item() log_dict['l_cls_neg'] = l_cls_neg.item() log_dict['l_reg'] = l_reg.item() elif cfg.MODE_TRAIN == 1: weight = gres[:, :, cfg.NUM_CLASSES + 4] # 这个可以判断正例 torch.Size([32, 845]) gtxy = gres[:, :, cfg.NUM_CLASSES:cfg.NUM_CLASSES + 2] gtwh = gres[:, :, cfg.NUM_CLASSES + 2:cfg.NUM_CLASSES + 4] ptxy_sigmoid = ptxy.sigmoid() # 这个需要归一化 _loss_val = x_bce(ptxy_sigmoid, gtxy, reduction="none") # _loss_val = F.binary_cross_entropy_with_logits(ptxy, gtxy, reduction="none") # _loss_val[mask_pos_2d].sum() 与这个等价 l_txty = torch.mean(torch.sum(torch.sum(_loss_val * weight.unsqueeze(-1), -1), -1) / nums_pos) _loss_val = F.smooth_l1_loss(ptwh, gtwh, reduction="none") l_twth = torch.mean(torch.sum(torch.sum(_loss_val * weight.unsqueeze(-1), -1), -1) / nums_pos) l_total = l_cls_pos + l_cls_neg + l_txty + l_twth log_dict['l_total'] = l_total.item() log_dict['l_cls_pos'] = l_cls_pos.item() log_dict['l_cls_neg'] = l_cls_neg.item() log_dict['l_xy'] = l_txty.item() log_dict['l_wh'] = l_twth.item() else: raise Exception('cfg.MODE_TRAIN = %s 不存在' % cfg.MODE_TRAIN) return l_total, log_dict
def forward(self, pyolos, targets, imgs_ts=None): ''' :param pyolos: torch.Size([32, 6, 14, 14]) [conf-1,class-20,box4] :param targets: :param imgs_ts: :return: ''' cfg = self.cfg device = pyolos.device batch, c, h, w = pyolos.shape # torch.Size([32, 13,13, 8]) # b,c,h,w -> b,c,hw -> b,hw,c torch.Size([32, 169, 8]) pyolos = pyolos.view(batch, c, -1).permute(0, 2, 1) s_ = 1 + cfg.NUM_CLASSES ptxywh = pyolos[..., s_:s_ + 4] # torch.Size([32, 169, 4]) # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 gdim = 1 + cfg.NUM_CLASSES + 4 + 1 + 4 gyolos = torch.empty((batch, h, w, gdim), device=device) # 每批会整体更新这里不需要赋0 for i, target in enumerate(targets): # batch遍历 gboxes_ltrb_b = target['boxes'] # ltrb glabels_b = target['labels'] ''' yolo4 1. 每层选一个匹配一个 anc与GT的IOU最大的一个 技巧gt的xy可调整成 格子偏移与pxy匹配 2. 其它的IOU>0.4忽略,除正例 3. reg损失: 解码预测 pxy.sigmoid exp(pwh*anc) -> 进行IOU loss 正例损失进行平均, 权重0.05 4. cls损失: label_smooth 标签平滑正则化, onehot* (1-0.01) + 0.01 /num_class pos_weight=0.5 loss_weight=0.5 * num_classes / 80 = 0.01875 5. conf损失: 整体权重0.4 忽略的 6. 每一层的损失全加起来 ''' gyolos[i] = fmatch4yolov1( gboxes_ltrb_b=gboxes_ltrb_b, glabels_b=glabels_b, grid=h, # 7 gdim=gdim, device=device, img_ts=imgs_ts[i], cfg=cfg, use_conf=True) '''可视化验证''' if cfg.IS_VISUAL: # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 gyolo_test = gyolos[i].clone() # torch.Size([32, 13, 13, 9]) gyolo_test = gyolo_test.view(-1, gdim) gconf_one = gyolo_test[:, 0] mask_pos = gconf_one == 1 # [169] # torch.Size([169, 4]) txywh_t = gyolo_test[:, 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 4] # 这里是修复所有的xy zpxy_t = txywh_t[:, :2] + f_mershgrid( h, w, is_rowcol=False).to(device) hw_ts = torch.tensor((h, w), device=device) zpxy = torch.true_divide(zpxy_t, hw_ts) zpwh = torch.exp(txywh_t[:, 2:]) / hw_ts zpxywh_pos = torch.cat([zpxy, zpwh], dim=-1)[mask_pos] from f_tools.pic.enhance.f_data_pretreatment4pil import f_recover_normalization4ts img_ts = f_recover_normalization4ts(imgs_ts[i]) from torchvision.transforms import functional as transformsF img_pil = transformsF.to_pil_image(img_ts).convert('RGB') import numpy as np img_np = np.array(img_pil) f_show_od_np4plt(img_np, gboxes_ltrb=gboxes_ltrb_b.cpu(), pboxes_ltrb=xywh2ltrb(zpxywh_pos.cpu()), is_recover_size=True, grids=(h, w)) gyolos = gyolos.view(batch, -1, gdim) # b,hw,7 gconf = gyolos[:, :, 0] # torch.Size([5, 169]) mask_pos = gconf > 0 # torch.Size([32, 169]) # mask_pos = gconf == 1 # yolo1 gt 写死是1 mask_neg = gconf == 0 nums_pos = (mask_pos.sum(-1).to( torch.float)).clamp(min=torch.finfo(torch.float16).eps) nums_neg = (mask_neg.sum(-1).to( torch.float)).clamp(min=torch.finfo(torch.float16).eps) pyolos_pos = pyolos[mask_pos] # torch.Size([32, 169, 13]) -> [nn, 13] gyolos_pos = gyolos[mask_pos] # torch.Size([32, 169, 13]) -> [nn, 13] ''' ---------------- 类别-cls损失 ---------------- ''' # # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 pcls_sigmoid = pyolos[:, :, 1:s_].sigmoid() # torch.Size([32, 169, 8]) gcls = gyolos[:, :, 1:s_] # torch.Size([32, 169, 13]) _loss_val = x_bce(pcls_sigmoid, gcls, reduction="none") l_cls = ((_loss_val.sum(-1) * mask_pos).sum(-1) / nums_pos).mean() # pcls_sigmoid_pos = pyolos_pos[:, 1:s_].sigmoid() # gcls_pos = gyolos_pos[:, 1:s_] # _loss_val = x_bce(pcls_sigmoid_pos, gcls_pos, reduction="none") # torch.Size([46, 3]) # torch.Size([46, 3]) -> val # l_cls = _loss_val.sum(-1).mean() ''' ---------------- 类别-conf损失 ---------------- ''' # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 pconf_sigmoid = pyolos[:, :, 0].sigmoid() # ------------ conf-mse ------------''' 666666 _loss_val = F.mse_loss(pconf_sigmoid, gconf, reduction="none") # 用MSE效果更好 l_conf_pos = ((_loss_val * mask_pos).sum(-1) / nums_pos).mean() * 5. l_conf_neg = ((_loss_val * mask_neg).sum(-1) / nums_pos).mean() * 1. # 效果一样 169:1 # pos_ = _loss_val[mask_pos] # l_conf_pos = pos_.mean() * 1 # l_conf_neg = _loss_val[mask_neg].mean() * 3 # ------------ conf_ohem ap26_26 ------------''' # _loss_val = x_bce(pconf_sigmoid, gconf) # mask_ignore = torch.logical_not(torch.logical_or(mask_pos, mask_neg)) # mask_neg_hard = f_ohem(_loss_val, nums_pos * 3, mask_pos=mask_pos, mash_ignore=mask_ignore) # l_conf_pos = ((_loss_val * mask_pos).sum(-1) / nums_pos).mean() * 3 # 正例越多反例越多 # l_conf_neg = ((_loss_val * mask_neg_hard).sum(-1) / nums_pos).mean() * 3 # ------------ focalloss ------------ # l_pos, l_neg = focalloss(pconf_sigmoid, gconf, mask_pos=mask_pos, is_debug=True, alpha=0.5) # l_conf_pos = (l_pos.sum(-1).sum(-1) / nums_pos).mean() # l_conf_neg = (l_neg.sum(-1).sum(-1) / nums_neg).mean() * 3 log_dict = {} ''' ----------------回归损失 xy采用bce wh采用mes----------------- ''' if cfg.MODE_TRAIN == 4: # ------------ iou损失 ------------ # 解码pxywh 计算预测与 GT 的 iou 作为 gconf # preg_pos = pyolos_pos[:, s_:s_ + 4] # # 解码yolo1 # pxy_pos_toff = preg_pos[..., :2].sigmoid() # pwh_pos = torch.exp(preg_pos[..., 2:]) # pzxywh = torch.cat([pxy_pos_toff, pwh_pos], -1) # 这里是归一化的 gt gltrb_pos = gyolos_pos[:, s_ + 4 + 1:s_ + 4 + 1 + 4] ptxywh = pyolos[..., s_:s_ + 4] pltrb_pos = boxes_decode4yolo1(ptxywh, h, w, cfg)[mask_pos] iou_zg = bbox_iou4one(pltrb_pos, gltrb_pos, is_giou=True) # iou_zg = bbox_iou4y(xywh2ltrb4ts(pzxywh), gltrb_pos_tx, GIoU=True) # print(iou_zg) l_reg = (1 - iou_zg).mean() * 5 ''' ---------------- loss完成 ----------------- ''' l_total = l_conf_pos + l_conf_neg + l_cls + l_reg log_dict['l_reg'] = l_reg.item() else: # ------------ mse+bce ------------ 666666 # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4 # torch.Size([32, 169, 13]) 9->实际是8 ptxty_sigmoid = pyolos[:, :, s_:s_ + 2].sigmoid() # 4:6 ptwth = pyolos[:, :, s_ + 2:s_ + 4] # 这里不需要归一 weight = gyolos[:, :, s_ + 4] # 这个是大小目标缩放比例 gtxty = gyolos[:, :, s_:s_ + 2] # torch.Size([5, 169, 2]) gtwth = gyolos[:, :, s_ + 2:s_ + 4] # _loss_val = x_bce(ptxty_sigmoid, gtxty, reduction="none") _loss_val = F.mse_loss(ptxty_sigmoid, gtxty, reduction="none") l_txty = ((_loss_val.sum(-1) * mask_pos * weight).sum(-1) / nums_pos).mean() _loss_val = F.mse_loss(ptwth, gtwth, reduction="none") l_twth = ((_loss_val.sum(-1) * mask_pos * weight).sum(-1) / nums_pos).mean() ''' ---------------- loss完成 ----------------- ''' l_total = l_conf_pos + l_conf_neg + l_cls + l_txty + l_twth log_dict['l_xy'] = l_txty.item() log_dict['l_wh'] = l_twth.item() log_dict['l_total'] = l_total.item() log_dict['l_conf_pos'] = l_conf_pos.item() log_dict['l_conf_neg'] = l_conf_neg.item() log_dict['l_cls'] = l_cls.item() log_dict['p_max'] = pconf_sigmoid.max().item() log_dict['p_min'] = pconf_sigmoid.min().item() log_dict['p_mean'] = pconf_sigmoid.mean().item() return l_total, log_dict
def forward(self, outs, targets, imgs_ts=None): ''' :param outs: torch.Size([2, 2125, 9]) :param targets: 'image_id': 413, 'size': tensor([500., 309.]) 'boxes': tensor([[0.31400, 0.31715, 0.71000, 0.60841]]), 'labels': tensor([1.]) :param imgs_ts: :return: ''' cfg = self.cfg device = outs.device batch, dim_total, pdim = outs.shape # back cls centerness ltrb positivesample iou area gdim = 1 + cfg.NUM_CLASSES + 1 + 4 + 1 + 1 + 1 gres = torch.empty((batch, dim_total, gdim), device=device) for i in range(batch): gboxes_ltrb_b = targets[i]['boxes'] glabels_b = targets[i]['labels'] import time # start = time.time() gres[i] = match4fcos_v2( gboxes_ltrb_b=gboxes_ltrb_b, glabels_b=glabels_b, gdim=gdim, pcos=outs, img_ts=imgs_ts[i], cfg=cfg, ) # gres[i] = match4fcos(gboxes_ltrb_b=gboxes_ltrb_b, # glabels_b=glabels_b, # gdim=gdim, # pcos=outs, # img_ts=imgs_ts[i], # cfg=cfg, ) # flog.debug('show_time---完成---%s--' % (time.time() - start)) s_ = 1 + cfg.NUM_CLASSES # outs = outs[:, :, :s_ + 1].sigmoid() mask_pos = gres[:, :, 0] == 0 # 背景为0 是正例 nums_pos = torch.sum(mask_pos, dim=-1) nums_pos = torch.max(nums_pos, torch.ones_like(nums_pos, device=device)) # back cls centerness ltrb positivesample iou(这个暂时无用) area [2125, 12] ''' ---------------- cls损失 计算全部样本,正反例,正例为框内本例---------------- ''' # obj_cls_loss = BCE_focal_loss() # 这里多一个背景一起算 pcls_sigmoid = outs[:, :, :s_].sigmoid() gcls = gres[:, :, :s_] # l_cls = torch.mean(obj_cls_loss(pcls_sigmoid, gcls) / nums_pos) l_cls_pos, l_cls_neg = focalloss_fcos(pcls_sigmoid, gcls) l_cls_pos = torch.mean( torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos) l_cls_neg = torch.mean( torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos) ''' ---------------- conf损失 只计算半径正例 center_ness---------------- ''' # 和 positive sample 算正例 mask_pp = gres[:, :, s_ + 1 + 4] == 1 pconf_sigmoid = outs[:, :, s_].sigmoid() # center_ness gcenterness = gres[:, :, s_] # (nn,1) # 使用centerness # _loss_val = x_bce(pconf_sigmoid, gcenterness, reduction="none") _loss_val = x_bce(pconf_sigmoid, torch.ones_like(pconf_sigmoid), reduction="none") # 用半径1 # 只算半径正例,提高准确性 l_conf = 5. * torch.mean( torch.sum(_loss_val * mask_pp.float(), dim=-1) / nums_pos) ''' ---------------- box损失 计算框内正例---------------- ''' # conf1 + cls3 + reg4 # poff_ltrb_exp = torch.exp(outs[:, :, s_:s_ + 4]) poff_ltrb = outs[:, :, s_:s_ + 4] # 这个全是特图的距离 全rule 或 exp # goff_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4] g_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4] # _loss_val = F.smooth_l1_loss(poff_ltrb, goff_ltrb, reduction='none') # _loss_val = F.mse_loss(poff_ltrb_exp, goff_ltrb, reduction='none') # l_reg = torch.sum(torch.sum(_loss_val, -1) * gconf * mask_pos.float(), -1) # l_reg = torch.mean(l_reg / nums_pos) # 这里是解析归一化图 # pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb, is_t=True) # p_ltrb_t_pos = pboxes_ltrb[mask_pos] # image_size_ts = torch.tensor(cfg.IMAGE_SIZE, device=device) # g_ltrb_t_pos = g_ltrb[mask_pos] * image_size_ts.repeat(2).view(1, -1) # iou = bbox_iou4one(p_ltrb_t_pos, g_ltrb_t_pos, is_giou=True) # 这里是解析归一化图 归一化与特图计算的IOU是一致的 pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb) p_ltrb_pos = pboxes_ltrb[mask_pos] g_ltrb_pos = g_ltrb[mask_pos] # iou = bbox_iou4one_2d(p_ltrb_pos, g_ltrb_pos, is_giou=True) iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True) # 使用 iou 与 1 进行bce debug iou.isnan().any() or iou.isinf().any() l_reg = 5 * torch.mean((1 - iou) * gcenterness[mask_pos]) # iou2 = bbox_iou4one_3d(pboxes_ltrb, g_ltrb, is_giou=True) # 2D 和 3D效果是一样的 # l_reg2 = torch.mean(torch.sum((1 - iou2) * gcenterness * mask_pos.float(), -1) / nums_pos) # _loss_val = x_bce(iou, giou, reduction="none") # l_iou = torch.mean(torch.sum(_loss_val * gconf * mask_pos.float(), dim=-1) / nums_pos) l_total = l_cls_pos + l_cls_neg + l_conf + l_reg log_dict = {} log_dict['l_total'] = l_total.item() log_dict['l_cls_pos'] = l_cls_pos.item() log_dict['l_cls_neg'] = l_cls_neg.item() log_dict['l_conf'] = l_conf.item() log_dict['l_reg'] = l_reg.item() # log_dict['l_iou_max'] = iou.max().item() return l_total, log_dict
def forward(self, outs, targets, imgs_ts=None): ''' :param outs: cls1+conf1+ltrb4 torch.Size([2, 2125, 9]) :param targets: 'image_id': 413, 'size': tensor([500., 309.]) 'boxes': tensor([[0.31400, 0.31715, 0.71000, 0.60841]]), 'labels': tensor([1.]) :param imgs_ts: :return: ''' cfg = self.cfg device = outs.device batch, dim_total, pdim = outs.shape # cls3 centerness1 ltrb4 positive_radius1 positive_ingt1 area1 3+1+4+1+1+1=11 gdim = cfg.NUM_CLASSES + 1 + 4 + 1 + 1 + 1 gres = torch.empty((batch, dim_total, gdim), device=device) nums_pos = [] for i in range(batch): gboxes_ltrb_b = targets[i]['boxes'] glabels_b = targets[i]['labels'] nums_pos.append(gboxes_ltrb_b.shape[0]) # import time # start = time.time() gres[i] = match4fcos_v3_noback( gboxes_ltrb_b=gboxes_ltrb_b, glabels_b=glabels_b, gdim=gdim, pcos=outs, img_ts=imgs_ts[i], cfg=cfg, ) # flog.debug('show_time---完成---%s--' % (time.time() - start)) # cls3 centerness1 ltrb4 positive_radius1 positive_ingt1 area1 mask_pos = gres[:, :, cfg.NUM_CLASSES + 1 + 4 + 1] == 1 # 框内正例 nums_pos = torch.tensor(nums_pos, device=device) ''' ---------------- cls损失 计算全部样本,正反例,正例为框内本例---------------- ''' # 框内3D正例 可以用 mask_pos_3d = gcls == 1 pcls_sigmoid = outs[:, :, :cfg.NUM_CLASSES].sigmoid() gcls = gres[:, :, :cfg.NUM_CLASSES] l_cls_pos, l_cls_neg = focalloss_fcos(pcls_sigmoid, gcls) l_cls_pos = torch.mean( torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos) l_cls_neg = torch.mean( torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos) ''' ---------------- conf损失 只计算半径正例 center_ness---------------- ''' # 半径正例 mask_pp = gres[:, :, cfg.NUM_CLASSES + 1 + 4] == 1 # 半径正例 pconf_sigmoid = outs[:, :, cfg.NUM_CLASSES].sigmoid() # center_ness gcenterness = gres[:, :, cfg.NUM_CLASSES] # (nn,1) # 使用centerness # 与 gcenterness 还是以1为准 # _loss_val = x_bce(pconf_sigmoid, gcenterness, reduction="none") _loss_val = x_bce(pconf_sigmoid, torch.ones_like(pconf_sigmoid), reduction="none") # 用半径1 # 只算半径正例,提高准确性 l_conf = 5. * torch.mean( torch.sum(_loss_val * mask_pp.float(), dim=-1) / nums_pos) ''' ---------------- box损失 计算框内正例---------------- ''' # cls3+ conf1+ reg4 poff_ltrb = outs[:, :, cfg.NUM_CLASSES + 1:cfg.NUM_CLASSES + 1 + 4] # 这个全是特图的距离 全rule 或 exp # goff_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4] g_ltrb = gres[:, :, cfg.NUM_CLASSES + 1:cfg.NUM_CLASSES + 1 + 4] # 这里是解析归一化图 归一化与特图计算的IOU是一致的 pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb) # 这里采用的是正例计算 直接平均 p_ltrb_pos = pboxes_ltrb[mask_pos] g_ltrb_pos = g_ltrb[mask_pos] iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True) # 使用 iou 与 1 进行bce debug iou.isnan().any() or iou.isinf().any() l_reg = 5 * torch.mean((1 - iou) * gcenterness[mask_pos]) l_total = l_cls_pos + l_cls_neg + l_conf + l_reg log_dict = {} log_dict['l_total'] = l_total.item() log_dict['l_cls_pos'] = l_cls_pos.item() log_dict['l_cls_neg'] = l_cls_neg.item() log_dict['l_conf'] = l_conf.item() log_dict['l_reg'] = l_reg.item() # log_dict['l_iou_max'] = iou.max().item() return l_total, log_dict