def forward(self, outs, targets, imgs_ts=None):
'''
模型尺寸list[20,10,5,3]
:param outs: torch.Size([2, 534, 7]) in 160 输出[]
:param targets:
'image_id': 413,
'size': tensor([500., 309.])
'boxes': tensor([[0.31400, 0.31715, 0.71000, 0.60841]]),
'labels': tensor([1.])
:param imgs_ts:
:return:
'''
cfg = self.cfg
device = outs.device
batch, dim_total, pdim = outs.shape
# 1 + cfg.NUM_CLASSES + 1 + 4 + cfg.NUM_KEYPOINTS * 2
# back cls centerness ltrb positivesample iou area
gdim = 1 + cfg.NUM_CLASSES + 1 + 4 + 1 + 1 + 1
gres = torch.empty((batch, dim_total, gdim), device=device)
for i in range(batch):
gboxes_ltrb_b = targets[i]['boxes']
glabels_b = targets[i]['labels']
gres[i] = match4fcos_v2(
gboxes_ltrb_b=gboxes_ltrb_b,
glabels_b=glabels_b,
gdim=gdim,
pcos=outs,
img_ts=imgs_ts[i],
cfg=cfg,
)
s_ = 1 + cfg.NUM_CLASSES
# outs = outs[:, :, :s_ + 1].sigmoid()
mask_pos = gres[:, :, 0] == 0 # 背景为0 是正例
nums_pos = torch.sum(mask_pos, dim=-1)
nums_pos = torch.max(nums_pos, torch.ones_like(nums_pos,
device=device))
# back cls centerness ltrb positivesample iou(这个暂时无用) area [2125, 12]
''' ---------------- cls损失 计算全部样本,正反例,正例为框内本例---------------- '''
# obj_cls_loss = BCE_focal_loss()
# 这里多一个背景一起算
pcls_sigmoid = outs[:, :, :s_].sigmoid()
gcls = gres[:, :, :s_]
# l_cls = torch.mean(obj_cls_loss(pcls_sigmoid, gcls) / nums_pos)
l_cls_pos, l_cls_neg = focalloss_fcos(pcls_sigmoid, gcls)
l_cls_pos = torch.mean(
torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos)
l_cls_neg = torch.mean(
torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos)
''' ---------------- conf损失 只计算半径正例 center_ness---------------- '''
# 和 positive sample 算正例
mask_pp = gres[:, :, s_ + 1 + 4] == 1
pconf_sigmoid = outs[:, :, s_].sigmoid() # center_ness
gcenterness = gres[:, :, s_] # (nn,1) # 使用centerness
# _loss_val = x_bce(pconf_sigmoid, gcenterness, reduction="none")
_loss_val = x_bce(pconf_sigmoid,
torch.ones_like(pconf_sigmoid),
reduction="none") # 用半径1
# 只算半径正例,提高准确性
l_conf = 5. * torch.mean(
torch.sum(_loss_val * mask_pp.float(), dim=-1) / nums_pos)
''' ---------------- box损失 计算框内正例---------------- '''
# conf1 + cls3 + reg4
poff_ltrb = outs[:, :, s_:s_ + 4] # 这个全是特图的距离 全rule 或 exp
# goff_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4]
g_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4]
# 这里是解析归一化图 归一化与特图计算的IOU是一致的
pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb)
p_ltrb_pos = pboxes_ltrb[mask_pos]
g_ltrb_pos = g_ltrb[mask_pos]
iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True)
# 使用 iou 与 1 进行bce debug iou.isnan().any() or iou.isinf().any()
l_reg = 5 * torch.mean((1 - iou) * gcenterness[mask_pos])
l_total = l_cls_pos + l_cls_neg + l_conf + l_reg
log_dict = {}
log_dict['l_total'] = l_total.item()
log_dict['l_cls_pos'] = l_cls_pos.item()
log_dict['l_cls_neg'] = l_cls_neg.item()
log_dict['l_conf'] = l_conf.item()
log_dict['l_reg'] = l_reg.item()
# log_dict['l_iou_max'] = iou.max().item()
return l_total, log_dict
def forward(self, pyolos, targets, imgs_ts=None):
'''
:param pyolos: torch.Size([2, 45, 13, 13])
:param targets:
:param imgs_ts:
:return:
'''
cfg = self.cfg
device = pyolos.device
batch, c, h, w = pyolos.shape # torch.Size([2, 45, 13, 13])
# [3, 40, 13, 13] -> [3, 8, 5, 13*13] -> [3, 169, 5, 8]
pyolos = pyolos.view(batch, 1 + cfg.NUM_CLASSES + 4, cfg.NUM_ANC, - 1).permute(0, 3, 2, 1).contiguous()
# [3, 169, 5, 8] -> [3, 169*5, 8]
pyolos = pyolos.view(batch, h * w * cfg.NUM_ANC, -1)
# pyolos = pyolos.view(batch, -1, s_ + 4)
preg = pyolos[..., 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 4] # torch.Size([2, 169, 5, 4])
pltrb = boxes_decode4yolo2_v2(preg, h, w, cfg) # 输出原图归一化 用于更新conf [2, 845, 4]
'''--------------gt匹配---------------'''
# conf-1, cls-1, txywh-4, weight-1, gltrb-4
if cfg.MODE_TRAIN == 99 or cfg.MODE_TRAIN == 98:
gdim = 1 + 1 + 4 + 1 + 4
gyolos = torch.empty((batch, h, w, cfg.NUM_ANC, gdim), device=device)
# 匹配GT
for i, target in enumerate(targets): # batch遍历
gboxes_ltrb_b = target['boxes'] # ltrb
glabels_b = target['labels']
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
if cfg.MODE_TRAIN == 99 or cfg.MODE_TRAIN == 98:
gyolos[i] = fmatch4yolov2_99(
gboxes_ltrb_b=gboxes_ltrb_b,
glabels_b=glabels_b,
grid=h, # 7 只有一层
gdim=gdim,
device=device,
cfg=cfg,
preg_b=preg[i],
img_ts=imgs_ts[i],
)
'''可视化验证'''
if cfg.IS_VISUAL:
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
gyolo_test = gyolos[i].clone() # torch.Size([32, 13, 13, 9])
gyolo_test = gyolo_test.view(-1, gdim)
gconf_one = gyolo_test[:, 0]
# mask_pos = torch.logical_or(gconf_one == 1, gconf_one == -1)
mask_pos_2d = gconf_one == 1
gtxywh = gyolo_test[:, 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 4]
# 这里是修复是 xy
_xy_grid = gtxywh[:, :2] + f_mershgrid(h, w, is_rowcol=False, num_repeat=cfg.NUM_ANC).to(device)
hw_ts = torch.tensor((h, w), device=device)
gtxywh[:, :2] = torch.true_divide(_xy_grid, hw_ts)
gtxywh = gtxywh[mask_pos_2d]
gtxywh[:, 2:4] = torch.exp(gtxywh[:, 2:]) / h # 原图归一化
from f_tools.pic.enhance.f_data_pretreatment4pil import f_recover_normalization4ts
img_ts = f_recover_normalization4ts(imgs_ts[i])
from torchvision.transforms import functional as transformsF
img_pil = transformsF.to_pil_image(img_ts).convert('RGB')
import numpy as np
img_np = np.array(img_pil)
f_show_od_np4plt(img_np, gboxes_ltrb=gboxes_ltrb_b.cpu()
, pboxes_ltrb=xywh2ltrb(gtxywh.cpu()), is_recover_size=True,
grids=(h, w))
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
# torch.Size([32, 13, 13, 5, 11]) -> [32, 13*13*5, 11] ->[2, 845, 11]
gyolos = gyolos.view(batch, -1, gdim)
gconf = gyolos[:, :, 0] # 正例使用1 torch.Size([32, 910])
# mask_pos_3d = gyolos[:, :, :1] > 0
# mask_neg_3d = gyolos[:, :, :1] == 0
mask_pos_2d = gconf > 0
mask_neg_2d = gconf == 0 # 忽略-1 不管
nums_pos = (mask_pos_2d.sum(-1).to(torch.float)).clamp(min=torch.finfo(torch.float16).eps)
nums_neg = (mask_neg_2d.sum(-1).to(torch.float)).clamp(min=torch.finfo(torch.float16).eps)
pyolos_pos = pyolos[mask_pos_2d] # torch.Size([32, 845, 8]) -> torch.Size([40, 8])
gyolos_pos = gyolos[mask_pos_2d] # torch.Size([32, 845, 13]) -> torch.Size([40, 8])
# [2, 845, 4] ->
# iou_zg = bbox_iou4one(pltrb, gyolos[..., 1 + 1 + 4 + 1:1 + 1 + 4 + 1 + 4], is_giou=True)
iou_zg = bbox_iou4one(pltrb, gyolos[..., 1 + 1 + 4 + 1:1 + 1 + 4 + 1 + 4], is_ciou=True)
''' ----------------cls损失---------------- '''
# pcls_sigmoid_pos = pyolos_pos[:, 1:1 + cfg.NUM_CLASSES].sigmoid()
pcls_pos = pyolos_pos[:, 1:1 + cfg.NUM_CLASSES]
gcls_pos = gyolos_pos[:, 1].long()
# torch.Size([3, 4]) ^^ tensor([2., 2., 3.])
_loss_val = F.cross_entropy(pcls_pos, gcls_pos, reduction="none")
# _loss_val = x_bce(pcls_sigmoid_pos, gcls_pos, reduction="none") # torch.Size([46, 3])
# torch.Size([46, 3]) -> val
l_cls = _loss_val.sum(-1).mean()
''' ----------------conf损失 ---------------- '''
pconf_sigmoid = pyolos[:, :, 0].sigmoid() # 这个需要归一化 torch.Size([3, 845])
# ------------conf-mse ------------
_loss_val = F.mse_loss(pconf_sigmoid, iou_zg, reduction="none") # 这理用的IOU
l_conf_pos = ((_loss_val * mask_pos_2d).sum(-1) / nums_pos).mean() * 5.
l_conf_neg = ((_loss_val * mask_neg_2d).sum(-1) / nums_neg).mean() * 1.
''' ---------------- box损失 ----------------- '''
pxy_pos_sigmoid = pyolos_pos[:, 1 + cfg.NUM_CLASSES:1 + cfg.NUM_CLASSES + 2].sigmoid() # 这个需要归一化
pwh_pos_scale = pyolos_pos[:, 1 + cfg.NUM_CLASSES + 2:1 + cfg.NUM_CLASSES + 4]
weight_pos = gyolos_pos[:, 1 + 1 + 4 + 1] # torch.Size([32, 845])
gtxy_pos = gyolos_pos[:, 1 + 1:1 + 1 + 2] # [nn]
gtwh_pos = gyolos_pos[:, 1 + 1 + 2:1 + 1 + 4]
_loss_val = x_bce(pxy_pos_sigmoid, gtxy_pos, reduction="none")
l_txty = (_loss_val.sum(-1) * weight_pos).mean()
_loss_val = F.mse_loss(pwh_pos_scale, gtwh_pos, reduction="none")
l_twth = (_loss_val.sum(-1) * weight_pos).mean()
''' ---------------- loss完成 ----------------- '''
log_dict = {}
l_total = l_conf_pos + l_conf_neg + l_cls + l_txty + l_twth
log_dict['l_total'] = l_total.item()
log_dict['l_xy'] = l_txty.item()
log_dict['l_wh'] = l_twth.item()
log_dict['l_conf_pos'] = l_conf_pos.item()
log_dict['l_conf_neg'] = l_conf_neg.item()
log_dict['l_cls'] = l_cls.item()
return l_total, log_dict
def forward(self, p_center, targets, imgs_ts=None):
'''
:param p_center:
:param targets: list
target['boxes'] = target['boxes'].to(device)
target['labels'] = target['labels'].to(device)
target['size'] = target['size']
target['image_id'] = int
:param imgs_ts:
:return:
'''
cfg = self.cfg
pcls, ptxy, ptwh = p_center
device = pcls.device
batch, c, h, w = pcls.shape
# b,c,h,w -> b,h,w,c -> b,h*w,c
pcls = pcls.permute(0, 2, 3, 1).contiguous().view(batch, -1, self.cfg.NUM_CLASSES)
ptxy = ptxy.permute(0, 2, 3, 1).contiguous().view(batch, -1, 2)
ptwh = ptwh.permute(0, 2, 3, 1).contiguous().view(batch, -1, 2)
fsize_wh = torch.tensor([h, w], device=device)
# num_class + txywh + weight + gt4 conf通过高斯生成 热力图层数表示类别索引
if cfg.NUM_KEYPOINTS > 0:
gdim = cfg.NUM_CLASSES + cfg.NUM_KEYPOINTS * 2 + 4 + 1 + 4
else:
gdim = cfg.NUM_CLASSES + 4 + 1 + 4
gres = torch.empty((batch, h, w, gdim), device=device)
# 匹配GT
for i, target in enumerate(targets):
# batch 遍历每一张图
gboxes_ltrb_b = targets[i]['boxes']
glabels_b = targets[i]['labels']
# 处理这张图的所有标签
gres[i] = match4center(gboxes_ltrb_b=gboxes_ltrb_b,
glabels_b=glabels_b,
fsize_wh=fsize_wh,
dim=gdim,
cfg=cfg,
)
if cfg.IS_VISUAL:
from f_tools.pic.enhance.f_data_pretreatment4pil import f_recover_normalization4ts
_img_ts = f_recover_normalization4ts(imgs_ts[i].clone())
from torchvision.transforms import functional as transformsF
img_pil = transformsF.to_pil_image(_img_ts).convert('RGB')
import numpy as np
# img_np = np.array(img_pil)
'''plt画图部分'''
from matplotlib import pyplot as plt
plt.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文标签
plt.rcParams['axes.unicode_minus'] = False
# 这里的热力图肯定的偏差 [128,128]
data_hot = torch.zeros_like(gres[i, :, :, 0]) # 只需要一层即可
for label in glabels_b.unique():
# print(ids2classes[str(int(label))])
# 类别合并输出
flog.debug(' %s', gres[i, :, :, 3:7][gres[i, :, :, (label - 1).long()] == 1])
torch.max(data_hot, gres[i, :, :, (label - 1).long()], out=data_hot) # 这里是类别合并
plt.imshow(data_hot.cpu())
plt.imshow(img_pil.resize(fsize_wh), alpha=0.7)
plt.colorbar()
# x,y表示横纵坐标，color表示颜色：'r':红 'b'：蓝色 等，marker:标记，edgecolors:标记边框色'r'、'g'等，s：size大小
boxes_xywh_cpu = ltrb2xywh(gboxes_ltrb_b).cpu()
fsize_cpu = fsize_wh.cpu()
xys_f = boxes_xywh_cpu[:, :2] * fsize_cpu
plt.scatter(xys_f[:, 0], xys_f[:, 1], color='r', s=5) # 红色
boxes_ltrb_cpu = gboxes_ltrb_b.cpu()
boxes_ltrb_f = boxes_ltrb_cpu * fsize_cpu.repeat(2)
current_axis = plt.gca()
for i, box_ltrb_f in enumerate(boxes_ltrb_f):
l, t, r, b = box_ltrb_f
# ltwh
current_axis.add_patch(plt.Rectangle((l, t), r - l, b - t, color='green', fill=False, linewidth=2))
# current_axis.text(l, t - 2, ids2classes[int(glabels[i])], size=8, color='white',
# bbox={'facecolor': 'green', 'alpha': 0.6})
plt.show()
gres = gres.reshape(batch, -1, gdim)
''' ---------------- cls损失 只计算正例---------------- '''
gcls = gres[:, :, :cfg.NUM_CLASSES]
# mask_pos_3d = gcls > 0 # torch.Size([3, 16384, 3])
# mask_neg_3d = gcls == 0
mask_pos_3d = gcls == 1 # 只有中心点为1正例 torch.Size([3, 16384, 3])
mask_neg_3d = gcls != 1
nums_pos = torch.sum(torch.sum(mask_pos_3d, dim=-1), dim=-1)
# mask_pos_2d = torch.any(mask_pos_3d, -1)
# focloss
pcls_sigmoid = pcls.sigmoid()
l_cls_pos, l_cls_neg = focalloss_center(pcls_sigmoid, gcls)
l_cls_pos = torch.mean(torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos)
l_cls_neg = torch.mean(torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos)
# l_cls_neg = l_cls_neg.sum(-1).sum(-1).mean() # 等价
''' ---------------- box损失 ----------------- '''
log_dict = {}
# num_class + txywh + weight + gt4
if cfg.MODE_TRAIN == 2: # iou
ptxywh = torch.cat([ptxy, ptwh], dim=-1)
pboxes_ltrb = boxes_decode4center(self.cfg, fsize_wh, ptxywh)
mask_pos_2d = torch.any(mask_pos_3d, -1) # torch.Size([16, 16384])
# torch.Size([16, 16384, 4]) -> torch.Size([19, 4])
p_ltrb_pos = pboxes_ltrb[mask_pos_2d]
g_ltrb_pos = gres[..., cfg.NUM_CLASSES + 4 + 1:cfg.NUM_CLASSES + 4 + 1 + 4][mask_pos_2d]
iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True)
l_reg = 5 * torch.mean(1 - iou)
l_total = l_cls_pos + l_cls_neg + l_reg
log_dict['l_total'] = l_total.item()
log_dict['l_cls_pos'] = l_cls_pos.item()
log_dict['l_cls_neg'] = l_cls_neg.item()
log_dict['l_reg'] = l_reg.item()
elif cfg.MODE_TRAIN == 1:
weight = gres[:, :, cfg.NUM_CLASSES + 4] # 这个可以判断正例 torch.Size([32, 845])
gtxy = gres[:, :, cfg.NUM_CLASSES:cfg.NUM_CLASSES + 2]
gtwh = gres[:, :, cfg.NUM_CLASSES + 2:cfg.NUM_CLASSES + 4]
ptxy_sigmoid = ptxy.sigmoid() # 这个需要归一化
_loss_val = x_bce(ptxy_sigmoid, gtxy, reduction="none")
# _loss_val = F.binary_cross_entropy_with_logits(ptxy, gtxy, reduction="none")
# _loss_val[mask_pos_2d].sum() 与这个等价
l_txty = torch.mean(torch.sum(torch.sum(_loss_val * weight.unsqueeze(-1), -1), -1) / nums_pos)
_loss_val = F.smooth_l1_loss(ptwh, gtwh, reduction="none")
l_twth = torch.mean(torch.sum(torch.sum(_loss_val * weight.unsqueeze(-1), -1), -1) / nums_pos)
l_total = l_cls_pos + l_cls_neg + l_txty + l_twth
log_dict['l_total'] = l_total.item()
log_dict['l_cls_pos'] = l_cls_pos.item()
log_dict['l_cls_neg'] = l_cls_neg.item()
log_dict['l_xy'] = l_txty.item()
log_dict['l_wh'] = l_twth.item()
else:
raise Exception('cfg.MODE_TRAIN = %s 不存在' % cfg.MODE_TRAIN)
return l_total, log_dict
def forward(self, pyolos, targets, imgs_ts=None):
'''
:param pyolos: torch.Size([32, 6, 14, 14]) [conf-1,class-20,box4]
:param targets:
:param imgs_ts:
:return:
'''
cfg = self.cfg
device = pyolos.device
batch, c, h, w = pyolos.shape # torch.Size([32, 13,13, 8])
# b,c,h,w -> b,c,hw -> b,hw,c torch.Size([32, 169, 8])
pyolos = pyolos.view(batch, c, -1).permute(0, 2, 1)
s_ = 1 + cfg.NUM_CLASSES
ptxywh = pyolos[..., s_:s_ + 4] # torch.Size([32, 169, 4])
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
gdim = 1 + cfg.NUM_CLASSES + 4 + 1 + 4
gyolos = torch.empty((batch, h, w, gdim),
device=device) # 每批会整体更新这里不需要赋0
for i, target in enumerate(targets): # batch遍历
gboxes_ltrb_b = target['boxes'] # ltrb
glabels_b = target['labels']
'''
yolo4
1. 每层选一个匹配一个 anc与GT的IOU最大的一个
技巧gt的xy可调整成 格子偏移与pxy匹配
2. 其它的IOU>0.4忽略,除正例
3. reg损失: 解码预测 pxy.sigmoid exp(pwh*anc) -> 进行IOU loss
正例损失进行平均, 权重0.05
4. cls损失:
label_smooth 标签平滑正则化, onehot* (1-0.01) + 0.01 /num_class
pos_weight=0.5
loss_weight=0.5 * num_classes / 80 = 0.01875
5. conf损失:
整体权重0.4 忽略的
6. 每一层的损失全加起来
'''
gyolos[i] = fmatch4yolov1(
gboxes_ltrb_b=gboxes_ltrb_b,
glabels_b=glabels_b,
grid=h, # 7
gdim=gdim,
device=device,
img_ts=imgs_ts[i],
cfg=cfg,
use_conf=True)
'''可视化验证'''
if cfg.IS_VISUAL:
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
gyolo_test = gyolos[i].clone() # torch.Size([32, 13, 13, 9])
gyolo_test = gyolo_test.view(-1, gdim)
gconf_one = gyolo_test[:, 0]
mask_pos = gconf_one == 1 # [169]
# torch.Size([169, 4])
txywh_t = gyolo_test[:, 1 + cfg.NUM_CLASSES:1 +
cfg.NUM_CLASSES + 4]
# 这里是修复所有的xy
zpxy_t = txywh_t[:, :2] + f_mershgrid(
h, w, is_rowcol=False).to(device)
hw_ts = torch.tensor((h, w), device=device)
zpxy = torch.true_divide(zpxy_t, hw_ts)
zpwh = torch.exp(txywh_t[:, 2:]) / hw_ts
zpxywh_pos = torch.cat([zpxy, zpwh], dim=-1)[mask_pos]
from f_tools.pic.enhance.f_data_pretreatment4pil import f_recover_normalization4ts
img_ts = f_recover_normalization4ts(imgs_ts[i])
from torchvision.transforms import functional as transformsF
img_pil = transformsF.to_pil_image(img_ts).convert('RGB')
import numpy as np
img_np = np.array(img_pil)
f_show_od_np4plt(img_np,
gboxes_ltrb=gboxes_ltrb_b.cpu(),
pboxes_ltrb=xywh2ltrb(zpxywh_pos.cpu()),
is_recover_size=True,
grids=(h, w))
gyolos = gyolos.view(batch, -1, gdim) # b,hw,7
gconf = gyolos[:, :, 0] # torch.Size([5, 169])
mask_pos = gconf > 0 # torch.Size([32, 169])
# mask_pos = gconf == 1 # yolo1 gt 写死是1
mask_neg = gconf == 0
nums_pos = (mask_pos.sum(-1).to(
torch.float)).clamp(min=torch.finfo(torch.float16).eps)
nums_neg = (mask_neg.sum(-1).to(
torch.float)).clamp(min=torch.finfo(torch.float16).eps)
pyolos_pos = pyolos[mask_pos] # torch.Size([32, 169, 13]) -> [nn, 13]
gyolos_pos = gyolos[mask_pos] # torch.Size([32, 169, 13]) -> [nn, 13]
''' ---------------- 类别-cls损失 ---------------- '''
# # conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
pcls_sigmoid = pyolos[:, :, 1:s_].sigmoid() # torch.Size([32, 169, 8])
gcls = gyolos[:, :, 1:s_] # torch.Size([32, 169, 13])
_loss_val = x_bce(pcls_sigmoid, gcls, reduction="none")
l_cls = ((_loss_val.sum(-1) * mask_pos).sum(-1) / nums_pos).mean()
# pcls_sigmoid_pos = pyolos_pos[:, 1:s_].sigmoid()
# gcls_pos = gyolos_pos[:, 1:s_]
# _loss_val = x_bce(pcls_sigmoid_pos, gcls_pos, reduction="none") # torch.Size([46, 3])
# torch.Size([46, 3]) -> val
# l_cls = _loss_val.sum(-1).mean()
''' ---------------- 类别-conf损失 ---------------- '''
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
pconf_sigmoid = pyolos[:, :, 0].sigmoid()
# ------------ conf-mse ------------''' 666666
_loss_val = F.mse_loss(pconf_sigmoid, gconf,
reduction="none") # 用MSE效果更好
l_conf_pos = ((_loss_val * mask_pos).sum(-1) / nums_pos).mean() * 5.
l_conf_neg = ((_loss_val * mask_neg).sum(-1) / nums_pos).mean() * 1.
# 效果一样 169:1
# pos_ = _loss_val[mask_pos]
# l_conf_pos = pos_.mean() * 1
# l_conf_neg = _loss_val[mask_neg].mean() * 3
# ------------ conf_ohem ap26_26 ------------'''
# _loss_val = x_bce(pconf_sigmoid, gconf)
# mask_ignore = torch.logical_not(torch.logical_or(mask_pos, mask_neg))
# mask_neg_hard = f_ohem(_loss_val, nums_pos * 3, mask_pos=mask_pos, mash_ignore=mask_ignore)
# l_conf_pos = ((_loss_val * mask_pos).sum(-1) / nums_pos).mean() * 3 # 正例越多反例越多
# l_conf_neg = ((_loss_val * mask_neg_hard).sum(-1) / nums_pos).mean() * 3
# ------------ focalloss ------------
# l_pos, l_neg = focalloss(pconf_sigmoid, gconf, mask_pos=mask_pos, is_debug=True, alpha=0.5)
# l_conf_pos = (l_pos.sum(-1).sum(-1) / nums_pos).mean()
# l_conf_neg = (l_neg.sum(-1).sum(-1) / nums_neg).mean() * 3
log_dict = {}
''' ----------------回归损失 xy采用bce wh采用mes----------------- '''
if cfg.MODE_TRAIN == 4:
# ------------ iou损失 ------------
# 解码pxywh 计算预测与 GT 的 iou 作为 gconf
# preg_pos = pyolos_pos[:, s_:s_ + 4]
# # 解码yolo1
# pxy_pos_toff = preg_pos[..., :2].sigmoid()
# pwh_pos = torch.exp(preg_pos[..., 2:])
# pzxywh = torch.cat([pxy_pos_toff, pwh_pos], -1)
# 这里是归一化的 gt
gltrb_pos = gyolos_pos[:, s_ + 4 + 1:s_ + 4 + 1 + 4]
ptxywh = pyolos[..., s_:s_ + 4]
pltrb_pos = boxes_decode4yolo1(ptxywh, h, w, cfg)[mask_pos]
iou_zg = bbox_iou4one(pltrb_pos, gltrb_pos, is_giou=True)
# iou_zg = bbox_iou4y(xywh2ltrb4ts(pzxywh), gltrb_pos_tx, GIoU=True)
# print(iou_zg)
l_reg = (1 - iou_zg).mean() * 5
''' ---------------- loss完成 ----------------- '''
l_total = l_conf_pos + l_conf_neg + l_cls + l_reg
log_dict['l_reg'] = l_reg.item()
else:
# ------------ mse+bce ------------ 666666
# conf-1, cls-num_class, txywh-4, weight-1, gltrb-4
# torch.Size([32, 169, 13]) 9->实际是8
ptxty_sigmoid = pyolos[:, :, s_:s_ + 2].sigmoid() # 4:6
ptwth = pyolos[:, :, s_ + 2:s_ + 4] # 这里不需要归一
weight = gyolos[:, :, s_ + 4] # 这个是大小目标缩放比例
gtxty = gyolos[:, :, s_:s_ + 2] # torch.Size([5, 169, 2])
gtwth = gyolos[:, :, s_ + 2:s_ + 4]
# _loss_val = x_bce(ptxty_sigmoid, gtxty, reduction="none")
_loss_val = F.mse_loss(ptxty_sigmoid, gtxty, reduction="none")
l_txty = ((_loss_val.sum(-1) * mask_pos * weight).sum(-1) /
nums_pos).mean()
_loss_val = F.mse_loss(ptwth, gtwth, reduction="none")
l_twth = ((_loss_val.sum(-1) * mask_pos * weight).sum(-1) /
nums_pos).mean()
''' ---------------- loss完成 ----------------- '''
l_total = l_conf_pos + l_conf_neg + l_cls + l_txty + l_twth
log_dict['l_xy'] = l_txty.item()
log_dict['l_wh'] = l_twth.item()
log_dict['l_total'] = l_total.item()
log_dict['l_conf_pos'] = l_conf_pos.item()
log_dict['l_conf_neg'] = l_conf_neg.item()
log_dict['l_cls'] = l_cls.item()
log_dict['p_max'] = pconf_sigmoid.max().item()
log_dict['p_min'] = pconf_sigmoid.min().item()
log_dict['p_mean'] = pconf_sigmoid.mean().item()
return l_total, log_dict
def forward(self, outs, targets, imgs_ts=None):
'''
:param outs: torch.Size([2, 2125, 9])
:param targets:
'image_id': 413,
'size': tensor([500., 309.])
'boxes': tensor([[0.31400, 0.31715, 0.71000, 0.60841]]),
'labels': tensor([1.])
:param imgs_ts:
:return:
'''
cfg = self.cfg
device = outs.device
batch, dim_total, pdim = outs.shape
# back cls centerness ltrb positivesample iou area
gdim = 1 + cfg.NUM_CLASSES + 1 + 4 + 1 + 1 + 1
gres = torch.empty((batch, dim_total, gdim), device=device)
for i in range(batch):
gboxes_ltrb_b = targets[i]['boxes']
glabels_b = targets[i]['labels']
import time
# start = time.time()
gres[i] = match4fcos_v2(
gboxes_ltrb_b=gboxes_ltrb_b,
glabels_b=glabels_b,
gdim=gdim,
pcos=outs,
img_ts=imgs_ts[i],
cfg=cfg,
)
# gres[i] = match4fcos(gboxes_ltrb_b=gboxes_ltrb_b,
# glabels_b=glabels_b,
# gdim=gdim,
# pcos=outs,
# img_ts=imgs_ts[i],
# cfg=cfg, )
# flog.debug('show_time---完成---%s--' % (time.time() - start))
s_ = 1 + cfg.NUM_CLASSES
# outs = outs[:, :, :s_ + 1].sigmoid()
mask_pos = gres[:, :, 0] == 0 # 背景为0 是正例
nums_pos = torch.sum(mask_pos, dim=-1)
nums_pos = torch.max(nums_pos, torch.ones_like(nums_pos,
device=device))
# back cls centerness ltrb positivesample iou(这个暂时无用) area [2125, 12]
''' ---------------- cls损失 计算全部样本,正反例,正例为框内本例---------------- '''
# obj_cls_loss = BCE_focal_loss()
# 这里多一个背景一起算
pcls_sigmoid = outs[:, :, :s_].sigmoid()
gcls = gres[:, :, :s_]
# l_cls = torch.mean(obj_cls_loss(pcls_sigmoid, gcls) / nums_pos)
l_cls_pos, l_cls_neg = focalloss_fcos(pcls_sigmoid, gcls)
l_cls_pos = torch.mean(
torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos)
l_cls_neg = torch.mean(
torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos)
''' ---------------- conf损失 只计算半径正例 center_ness---------------- '''
# 和 positive sample 算正例
mask_pp = gres[:, :, s_ + 1 + 4] == 1
pconf_sigmoid = outs[:, :, s_].sigmoid() # center_ness
gcenterness = gres[:, :, s_] # (nn,1) # 使用centerness
# _loss_val = x_bce(pconf_sigmoid, gcenterness, reduction="none")
_loss_val = x_bce(pconf_sigmoid,
torch.ones_like(pconf_sigmoid),
reduction="none") # 用半径1
# 只算半径正例,提高准确性
l_conf = 5. * torch.mean(
torch.sum(_loss_val * mask_pp.float(), dim=-1) / nums_pos)
''' ---------------- box损失 计算框内正例---------------- '''
# conf1 + cls3 + reg4
# poff_ltrb_exp = torch.exp(outs[:, :, s_:s_ + 4])
poff_ltrb = outs[:, :, s_:s_ + 4] # 这个全是特图的距离 全rule 或 exp
# goff_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4]
g_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4]
# _loss_val = F.smooth_l1_loss(poff_ltrb, goff_ltrb, reduction='none')
# _loss_val = F.mse_loss(poff_ltrb_exp, goff_ltrb, reduction='none')
# l_reg = torch.sum(torch.sum(_loss_val, -1) * gconf * mask_pos.float(), -1)
# l_reg = torch.mean(l_reg / nums_pos)
# 这里是解析归一化图
# pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb, is_t=True)
# p_ltrb_t_pos = pboxes_ltrb[mask_pos]
# image_size_ts = torch.tensor(cfg.IMAGE_SIZE, device=device)
# g_ltrb_t_pos = g_ltrb[mask_pos] * image_size_ts.repeat(2).view(1, -1)
# iou = bbox_iou4one(p_ltrb_t_pos, g_ltrb_t_pos, is_giou=True)
# 这里是解析归一化图 归一化与特图计算的IOU是一致的
pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb)
p_ltrb_pos = pboxes_ltrb[mask_pos]
g_ltrb_pos = g_ltrb[mask_pos]
# iou = bbox_iou4one_2d(p_ltrb_pos, g_ltrb_pos, is_giou=True)
iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True)
# 使用 iou 与 1 进行bce debug iou.isnan().any() or iou.isinf().any()
l_reg = 5 * torch.mean((1 - iou) * gcenterness[mask_pos])
# iou2 = bbox_iou4one_3d(pboxes_ltrb, g_ltrb, is_giou=True) # 2D 和 3D效果是一样的
# l_reg2 = torch.mean(torch.sum((1 - iou2) * gcenterness * mask_pos.float(), -1) / nums_pos)
# _loss_val = x_bce(iou, giou, reduction="none")
# l_iou = torch.mean(torch.sum(_loss_val * gconf * mask_pos.float(), dim=-1) / nums_pos)
l_total = l_cls_pos + l_cls_neg + l_conf + l_reg
log_dict = {}
log_dict['l_total'] = l_total.item()
log_dict['l_cls_pos'] = l_cls_pos.item()
log_dict['l_cls_neg'] = l_cls_neg.item()
log_dict['l_conf'] = l_conf.item()
log_dict['l_reg'] = l_reg.item()
# log_dict['l_iou_max'] = iou.max().item()
return l_total, log_dict
def forward(self, outs, targets, imgs_ts=None):
'''
:param outs: cls1+conf1+ltrb4 torch.Size([2, 2125, 9])
:param targets:
'image_id': 413,
'size': tensor([500., 309.])
'boxes': tensor([[0.31400, 0.31715, 0.71000, 0.60841]]),
'labels': tensor([1.])
:param imgs_ts:
:return:
'''
cfg = self.cfg
device = outs.device
batch, dim_total, pdim = outs.shape
# cls3 centerness1 ltrb4 positive_radius1 positive_ingt1 area1 3+1+4+1+1+1=11
gdim = cfg.NUM_CLASSES + 1 + 4 + 1 + 1 + 1
gres = torch.empty((batch, dim_total, gdim), device=device)
nums_pos = []
for i in range(batch):
gboxes_ltrb_b = targets[i]['boxes']
glabels_b = targets[i]['labels']
nums_pos.append(gboxes_ltrb_b.shape[0])
# import time
# start = time.time()
gres[i] = match4fcos_v3_noback(
gboxes_ltrb_b=gboxes_ltrb_b,
glabels_b=glabels_b,
gdim=gdim,
pcos=outs,
img_ts=imgs_ts[i],
cfg=cfg,
)
# flog.debug('show_time---完成---%s--' % (time.time() - start))
# cls3 centerness1 ltrb4 positive_radius1 positive_ingt1 area1
mask_pos = gres[:, :, cfg.NUM_CLASSES + 1 + 4 + 1] == 1 # 框内正例
nums_pos = torch.tensor(nums_pos, device=device)
''' ---------------- cls损失 计算全部样本,正反例,正例为框内本例---------------- '''
# 框内3D正例 可以用 mask_pos_3d = gcls == 1
pcls_sigmoid = outs[:, :, :cfg.NUM_CLASSES].sigmoid()
gcls = gres[:, :, :cfg.NUM_CLASSES]
l_cls_pos, l_cls_neg = focalloss_fcos(pcls_sigmoid, gcls)
l_cls_pos = torch.mean(
torch.sum(torch.sum(l_cls_pos, -1), -1) / nums_pos)
l_cls_neg = torch.mean(
torch.sum(torch.sum(l_cls_neg, -1), -1) / nums_pos)
''' ---------------- conf损失 只计算半径正例 center_ness---------------- '''
# 半径正例
mask_pp = gres[:, :, cfg.NUM_CLASSES + 1 + 4] == 1 # 半径正例
pconf_sigmoid = outs[:, :, cfg.NUM_CLASSES].sigmoid() # center_ness
gcenterness = gres[:, :, cfg.NUM_CLASSES] # (nn,1) # 使用centerness
# 与 gcenterness 还是以1为准
# _loss_val = x_bce(pconf_sigmoid, gcenterness, reduction="none")
_loss_val = x_bce(pconf_sigmoid,
torch.ones_like(pconf_sigmoid),
reduction="none") # 用半径1
# 只算半径正例,提高准确性
l_conf = 5. * torch.mean(
torch.sum(_loss_val * mask_pp.float(), dim=-1) / nums_pos)
''' ---------------- box损失 计算框内正例---------------- '''
# cls3+ conf1+ reg4
poff_ltrb = outs[:, :, cfg.NUM_CLASSES + 1:cfg.NUM_CLASSES + 1 +
4] # 这个全是特图的距离 全rule 或 exp
# goff_ltrb = gres[:, :, s_ + 1:s_ + 1 + 4]
g_ltrb = gres[:, :, cfg.NUM_CLASSES + 1:cfg.NUM_CLASSES + 1 + 4]
# 这里是解析归一化图 归一化与特图计算的IOU是一致的
pboxes_ltrb = boxes_decode4fcos(self.cfg, poff_ltrb)
# 这里采用的是正例计算 直接平均
p_ltrb_pos = pboxes_ltrb[mask_pos]
g_ltrb_pos = g_ltrb[mask_pos]
iou = bbox_iou4one(p_ltrb_pos, g_ltrb_pos, is_giou=True)
# 使用 iou 与 1 进行bce debug iou.isnan().any() or iou.isinf().any()
l_reg = 5 * torch.mean((1 - iou) * gcenterness[mask_pos])
l_total = l_cls_pos + l_cls_neg + l_conf + l_reg
log_dict = {}
log_dict['l_total'] = l_total.item()
log_dict['l_cls_pos'] = l_cls_pos.item()
log_dict['l_cls_neg'] = l_cls_neg.item()
log_dict['l_conf'] = l_conf.item()
log_dict['l_reg'] = l_reg.item()
# log_dict['l_iou_max'] = iou.max().item()
return l_total, log_dict