def __call__(self, p, targets): # predictions, targets, model
device = targets.device
lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)
tcls, tbox, indices, anchors = self.build_targets(p, targets) # targets
# Losses
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, EIoU=True) # iou(prediction, target)
if self.g2>0 :# Focal-EIOU LOSS https://arxiv.org/abs/2101.08158
lbox += ((bbox_iou(pbox.T, tbox[i], x1y1x2y2=False)** g2)*(1 - iou)).mean()
else:
lbox += (1.0 - iou).mean() # iou loss
# Objectness
tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
# Classification
if self.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets
t[range(n), tcls[i]] = self.cp
lcls += self.BCEcls(ps[:, 5:], t) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
obji = self.BCEobj(pi[..., 4], tobj)
lobj += obji * self.balance[i] # obj loss
if self.autobalance:
self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()
if self.autobalance:
self.balance = [x / self.balance[self.ssi] for x in self.balance]
lbox *= self.hyp['box']
lobj *= self.hyp['obj']
lcls *= self.hyp['cls']
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls
return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()
def compute_loss(p, targets, model): # predictions, targets, model
device = targets.device
lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)
tcls, tbox, indices, anchors = build_targets(p, targets, model) # targets
h = model.hyp # hyperparameters
# Define criteria
BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) # weight=model.class_weights)
BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))
# Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3
cp, cn = smooth_BCE(eps=0.0)
# Focal loss
g = h['fl_gamma'] # focal loss gamma
if g > 0:
BCEcls, BCEobj = QFocalLoss(BCEcls, g), QFocalLoss(BCEobj, g)
# BCEobj = QFocalLoss(BCEobj, g)
# Losses
nt = 0 # number of targets
balance = [4.0, 1.0, 0.3, 0.1, 0.03] # P3-P7
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
nt += n # cumulative targets
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean() # iou loss
# Objectness
tobj[b, a, gj, gi] = (1.0 - model.gr) + model.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
# Classification
if model.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:], cn, device=device) # targets
t[range(n), tcls[i]] = cp
lcls += BCEcls(ps[:, 5:], t) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss
lbox *= h['box']
lobj *= h['obj']
lcls *= h['cls']
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls
return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()
def __call__(self, p, targets): # predictions, targets, model
device = targets.device
lcls1, lcls2, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device),\
torch.zeros(1, device=device), torch.zeros(1, device=device)
tcls1, tcls2, tbox, indices, anchors = self.build_targets(p, targets) # targets
# Losses
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean() # iou loss
# Objectness
tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
# Classification
if self.nc1 > 1 or self.nc2 > 1: # cls loss (only if multiple classes) # edit
# print(f"n: {n}, ps: {ps.shape}, tcls1: {tcls1}, tcls2: {tcls2}, lcls1: {lcls1}, lcls2: {lcls2}") # todo
t1 = torch.full_like(ps[:, 5:5+self.nc1], self.cn, device=device) # targets
t1[range(n), tcls1[i]] = self.cp
lcls1 += self.BCEcls(ps[:, 5:5+self.nc1], t1) # BCE
t2 = torch.full_like(ps[:, 5+self.nc1:], self.cn, device=device) # targets
t2[range(n), tcls2[i]] = self.cp
lcls2 += self.BCEcls(ps[:, 5+self.nc1:], t2) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
obji = self.BCEobj(pi[..., 4], tobj)
lobj += obji * self.balance[i] # obj loss
if self.autobalance:
self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()
if self.autobalance:
self.balance = [x / self.balance[self.ssi] for x in self.balance]
lbox *= self.hyp['box']
lobj *= self.hyp['obj']
lcls1 *= self.hyp['cls'] # edit
lcls2 *= self.hyp['cls'] # edit
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls1 + lcls2 # edit
return loss * bs, torch.cat((lbox, lobj, lcls1, lcls2, loss)).detach() # edit
def segment_loss(self, preds, targets,
masks): # predictions, targets, model
"""
proto_out:[batch-size, mask_dim, mask_hegiht, mask_width]
masks:[batch-size * num_objs, image_height, image_width]
每张图片objects数量不同，到时候处理时填充不足的
"""
p = preds[0]
proto_out = preds[1]
# print(proto_out.shape)
# batch_size, mask_dim, mask_h, mask_w
mask_h, mask_w = proto_out.shape[2:]
proto_out = proto_out.permute(0, 2, 3, 1)
device = targets.device
lcls, lbox, lobj, lseg = torch.zeros(1, device=device), torch.zeros(
1, device=device), torch.zeros(1, device=device), torch.zeros(
1, device=device)
tcls, tbox, indices, anchors, tidxs, xywh = self.build_targets_(
p, targets) # targets
# Losses
# savei = 0
total_pos = 0
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
ps = pi[b, a, gj,
gi] # prediction subset corresponding to targets
# Regression
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2)**2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False,
CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean() # iou loss
# Objectness
tobj[b, a, gj,
gi] = (1.0 -
self.gr) + self.gr * iou.detach().clamp(0).type(
tobj.dtype) # iou ratio
# Classification
if self.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 37:], self.cn,
device=device) # targets
t[range(n), tcls[i]] = self.cp
lcls += self.BCEcls(ps[:, 37:], t) # BCE
# mask proto
# [mask_h, mask_w, mask_dim] @ [mask_dim, num_pos]
# proto_temp = proto_out[b]
# print(proto_temp.shape)
# print(ps[:, 5:37].shape)
# pred_mask = torch.clamp(
# proto_temp @ ps[:, 5:37].tanh().T,
# 0,
# # mask_h, mask_w, num_pos
# 1)
# print(pred_mask.shape)
# # num_pos, image_h, image_w
mask_gt = masks[tidxs[i]]
# for k in range(mask_gt.shape[0]):
# cv2.imshow('p', mask_gt[k].cpu().numpy())
# cv2.waitKey(0)
# print(len(mask_gt[mask_gt > 0]))
downsampled_masks = F.interpolate(mask_gt[None, :],
(mask_h, mask_w),
mode='bilinear',
align_corners=False)
# mask_h, mask_w, num_pos
# downsampled_masks = downsampled_masks.squeeze().permute(
# 1, 2, 0).contiguous()
# lseg += F.binary_cross_entropy(pred_mask,
# downsampled_masks,
# reduce='sum')
# print(b.unique())
for bi in b.unique():
# print(b, bi)
index = b == bi
# print(index.sum())
total_pos += index.sum()
bm, am, gjm, gim = b[index], a[index], gj[index], gi[index]
# print(downsampled_masks.squeeze().shape)
# print(index.shape)
# mask_gti = downsampled_masks.squeeze()[index] # squeeze()会将维度1的轴都去掉
mask_gti = downsampled_masks[0][index]
mask_gti = mask_gti.permute(1, 2, 0).contiguous()
mxywh = xywh[i][index]
mw, mh = mxywh[:, 2:].T
mw, mh = mw / pi.shape[3], mh / pi.shape[2]
# print(mxywh.shape)
mxywh = mxywh / torch.tensor(
pi.shape,
device=mxywh.device)[[3, 2, 3, 2]] * torch.tensor(
[mask_w, mask_h, mask_w, mask_h],
device=mxywh.device) # psi = ps[b == bi]
mxyxy = xywh2xyxy(mxywh)
psi = pi[bm, am, gjm, gim]
# psi.tanh().cpu().detach().numpy())
pred_maski = proto_out[bi] @ psi[:, 5:37].tanh().T
# maskssss = crop(mask_gti, mxyxy)
# for k in range(maskssss.shape[-1]):
# cv2.imshow('p', cv2.resize(maskssss[:, :, k].cpu().numpy(), (640, 640)))
# cv2.waitKey(0)
# pred_maski = proto_out[bi] @ psi.tanh().T
# np.savetxt(
# f'mask_c/{savei}.txt',
# pred_maski[:, :, 0].sigmoid().cpu().detach().numpy())
# pred_maski = pred_maski.sigmoid()
# savei += 1
# print(pred_maski.shape)
# print(mask_gti.shape)
# print(len(mask_gti[mask_gti > 0]))
# cv2.imshow(
# 'p',
# mask_gti[:, :, 0].cpu().numpy().astype(np.uint8) * 255)
# if cv2.waitKey(0) == ord('q'):
# exit()
# lseg += nn.MSELoss(reduction='mean')(pred_maski, mask_gti)
lseg_ = F.binary_cross_entropy_with_logits(
pred_maski, mask_gti, reduction='none') * 6.125
lseg_ = crop(lseg_, mxyxy)
# print(lseg_.shape)
lseg_ = lseg_.mean(dim=(0, 1)) / mw / mh
lseg += torch.sum(lseg_)
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
obji = self.BCEobj(pi[..., 4], tobj)
lobj += obji * self.balance[i] # obj loss
if self.autobalance:
self.balance[i] = self.balance[
i] * 0.9999 + 0.0001 / obji.detach().item()
if self.autobalance:
self.balance = [x / self.balance[self.ssi] for x in self.balance]
lbox *= self.hyp['box']
lobj *= self.hyp['obj']
lcls *= self.hyp['cls']
lseg *= (self.hyp['box'] / 2)
lseg /= total_pos
# lseg *= 6.125
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls + lseg
return loss * bs, torch.cat((lbox, lobj, lcls, lseg, loss)).detach()
def __call__(self, p, targets): # predictions, targets, model
edges = self.hyp['edges']
device = targets.device
lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(
1, device=device), torch.zeros(1, device=device)
tcls, tbox, tpath, indices, anchors = self.build_targets(
p, targets, edges) # targets
# Losses #p.shape:[3,2,3,160,160,89]
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
ps = pi[b, a, gj,
gi] # prediction subset corresponding to targets
# Regression
if edges == 2:
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2)**2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=True,
DIoU=True) # iou(prediction, target)
elif edges == 4:
direction = torch.Tensor([-1, 1, 1, 1, 1, -1, -1,
-1]).to(ps.device)
pbox = (ps[:, :8].sigmoid() * 2)**2 * anchors[i].repeat(
1, 4) * direction
iou = ppoly_iou(pbox, tpath[i])
else:
pbox = (ps[:, :edges * 2].sigmoid() * 4 -
2)**2 * anchors[i].repeat(1, edges)
iou = ppoly_iou(pbox, tpath[i])
lbox += (1.0 - iou).mean() # iou loss
# Objectness
tobj[b, a, gj,
gi] = (1.0 -
self.gr) + self.gr * iou.detach().clamp(0).type(
tobj.dtype) # iou ratio
# Classification
if self.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, edges * 2 + 1:],
self.cn,
device=device) # targets
t[range(n), tcls[i]] = self.cp
lcls += self.BCEcls(ps[:, edges * 2 + 1:], t) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
obji = self.BCEobj(pi[..., 2 * edges], tobj)
lobj += obji * self.balance[i] # obj loss
if self.autobalance:
self.balance[i] = self.balance[
i] * 0.9999 + 0.0001 / obji.detach().item()
if self.autobalance:
self.balance = [x / self.balance[self.ssi] for x in self.balance]
lbox *= self.hyp['box']
lobj *= self.hyp['obj']
lcls *= self.hyp['cls']
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls
return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()
def compute_loss_refinenet(p,targets,boxes,model):
device = targets.device
all_batch_target = build_targets_forbatch([320,320],targets,boxes)
b_boxes = torch.cat(boxes,0)
indices,tbox,tcls = build_targets_forlayer(p, all_batch_target)
bs = p[0][...,0].shape[0]
# # print(bs)
lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)
h = model.hyp # hyperparameters
# Define criteria
BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['cls_pw']])).to(device)
BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['obj_pw']])).to(device)
# Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3
cp, cn = smooth_BCE(eps=0.0)
# Focal loss
g = h['fl_gamma'] # focal loss gamma
if g > 0:
BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)
# Losses
nt = 0 # number of targets
no = len(p) # number of outputs
balance = [4.0, 1.0, 0.4] if no == 3 else [4.0, 1.0, 0.4, 0.1] # P3-5 or P3-6
for i, pi in enumerate(p): # layer index, layer predictions
b, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
# print(n)
if n:
nt += n # cumulative targets
ps = pi[b, gj, gi] # prediction subset corresponding to targets
# Regression
if True:
pxy = ps[:, :2].sigmoid() * 2. - 0.5
# print(pi.shape)
pwh = (ps[:, 2:4].sigmoid()*torch.tensor([pi.shape[1],pi.shape[1]]).to(device)).to(device)
pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box
# iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, DIoU=True,CIoU=True) # iou(prediction, target)
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False,CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean()
# Objectness
tobj[b, gj, gi] = (1.0 - model.gr) + model.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
# print(torch.max(tobj),torch.max(pi[..., 4]))
# Classification
if model.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:], cn, device=device) # targets
t[range(n), tcls[i]] = cp
lcls += BCEcls(ps[:, 5:], t) # BCE
lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss
s = 3 / no # output count scaling
lbox *= h['box'] * s*0.5
lobj *= h['obj'] * s * (1.4 if no == 4 else 1.)*0.5
lcls *= h['cls'] * s*0.5
bs = tobj.shape[0] # batch size
# else:
loss = lbox +lobj+lcls
loss = loss*0.01
return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()
# p:N*85*w*h
# target:K*6
# boxes: N*4
return 0
def spider_sense(headDet, weapDet, im0, thres):
print("Starting Spider-Sense")
detections = [False, False, 0, False]
# print(len(headDet), headDet)
# print(len(weapDet), weapDet)
headThres = {2: 0.21, 3: 0.15}
# remove detections that are incredibly overlapping and have same class
for i in range(0, len(headDet[-1])):
if headDet[-1][i] is False:
continue
for j in range(i + 1, len(headDet[-1])):
if headDet[-1][j] is False:
continue
iou = bbox_iou(headDet[-1][i], headDet[-1][j])
print(iou)
if iou >= 0.85:
# print("clone detect")
if headDet[-1][j][4] > headDet[-1][i][4]:
headDet[-1][i] = False
break
else:
headDet[-1][j] = False
for i in range(0, len(weapDet[-1])):
if weapDet[-1][i] is False:
continue
for j in range(i + 1, len(weapDet[-1])):
if weapDet[-1][j] is False:
continue
if weapDet[-1][i][5] != weapDet[-1][j][5]:
continue
iou = bbox_iou(weapDet[-1][i], weapDet[-1][j])
print(iou)
if iou >= 0.85:
print("clone detect")
if weapDet[-1][j][4] > weapDet[-1][i][4]:
weapDet[-1][i] = False
break
else:
weapDet[-1][j] = False
# Checking for head width and weapons and doing context check if valid
if len(headDet[-1]) and len(headDet) == 5:
detections[2] += len(headDet[-1])
for detection in headDet[-1]:
if type(detection) == bool:
continue
width = float((detection[2] - detection[0]) / im0.shape[1])
if width >= headThres[thres]:
context = 0
tempDet = detection
ic(len(headDet))
for i in range(3, -1, -1): # each frame
for j in range(0, len(headDet[i])): # each detection in frame
print(tempDet)
print(headDet[i][j])
if ic(bbox_iou(tempDet, headDet[i][j], DIoU=True)) >= 0.1542:
context += 1
tempDet = headDet[i][j]
break
if context != (4 - i):
break
ic(context)
if context >= 3:
detections[0] = True
break
print("WIDTH:", width)
elif len(headDet) < 5:
for detection in headDet[-1]:
width = float((detection[2] - detection[0]) / im0.shape[1])
if width >= headThres[thres]:
detections[3] = True
break
noContext = 0
if len(weapDet[-1]) > 0 and len(weapDet) == 5:
for detection in weapDet[-1]:
context = 0
tempDet = detection
ic(len(weapDet))
for i in range(3, -1, -1): # each frame
for j in range(0, len(weapDet[i])): # each detection in frame
print(tempDet)
print(weapDet[i][j])
if tempDet[5] == weapDet[i][j][5] and ic(bbox_iou(tempDet, weapDet[i][j], DIoU=True)) >= 0.1542:
context += 1
tempDet = weapDet[i][j]
break
if context < (3-i):
break
ic(context)
if context >= 3:
detections[1] = True
break
return detections
def compute_loss(p, targets, model, imgs=None): # predictions, targets, model
# 可视化target
if imgs != None:
vis_bbox(imgs, targets)
device = targets.device
lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(
1, device=device), torch.zeros(1, device=device)
tcls, tbox, indices, anchors, ttar = build_targets(
p, targets, model
) # targets: classid, box ratio, image, anchor, grid indices, anchors, box
# 可视化anchor匹配关系
if imgs != None:
vis_match(imgs, targets, tcls, tbox, indices, anchors, p, ttar)
h = model.hyp # hyperparameters
# Define criteria
BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(
[h['cls_pw']], device=device)) # weight=model.class_weights)
BCEobj = nn.BCEWithLogitsLoss(
pos_weight=torch.tensor([h['obj_pw']], device=device))
# Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3
cp, cn = smooth_BCE(eps=0.0)
# Focal loss
g = h['fl_gamma'] # focal loss gamma
if g > 0:
BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)
# Losses
nt = 0 # number of targets
no = len(p) # number of outputs channels
balance = [4.0, 1.0, 0.4] if no == 3 else [4.0, 1.0, 0.4, 0.1
] # P3-5 or P3-6
# 遍历每个预测输出
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
nt += n # cumulative targets
# 取出对应位置预测值
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression GIOU
pxy = ps[:, :2].sigmoid() * 2. - 0.5 # xy offset
pwh = (ps[:, 2:4].sigmoid() *
2)**2 * anchors[i] # 其没有采用exp操作，而是直接乘上anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False,
CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean() # iou loss
# Objectness 有物体的conf分支权重
# 类似fcos和yolov2，虽然我们引入了大量正样本anchor，但是不同anchor和gt bbox匹配度是不一样，预测框和gt bbox 的匹配度也不一样，如果权重设置一样肯定不是最优的，
# 故作者将预测框和bbox的giou作为权重乘到conf分支，用于表征预测质量。
tobj[b, a, gj,
gi] = (1.0 -
model.gr) + model.gr * iou.detach().clamp(0).type(
tobj.dtype) # giou ratio
# Classification
if model.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:], cn,
device=device) # 建立预测targets的所有类别的大小
t[range(n), tcls[i]] = cp # 对应预测目标位置置为1
lcls += BCEcls(ps[:, 5:], t) # BCE 每个类单独计算loss
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss
s = 3 / no # output count scaling
lbox *= h['box'] * s
lobj *= h['obj'] * s * (1.4 if no == 4 else 1.)
lcls *= h['cls'] * s
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls
return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()
def compute_loss(p, targets, model): # predictions, targets, model
device = targets.device
lcls, lbox, lobj, llandmark = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)
tcls, tbox, indices, anchors = build_targets(p, targets, model) # targets
h = model.hyp # hyperparameters
# Define criteria
BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['cls_pw']])).to(device)
BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['obj_pw']])).to(device)
MSElandmarks = nn.MSELoss()
# Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3
cp, cn = smooth_BCE(eps=0.0)
# Focal loss
g = h['fl_gamma'] # focal loss gamma
if g > 0:
BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)
# Losses
nt = 0 # number of targets
no = len(p) # number of outputs
balance = [4.0, 1.0, 0.4] if no == 3 else [4.0, 1.0, 0.4, 0.1] # P3-5 or P3-6
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
if n:
nt += n # cumulative targets
# print(pi.shape)
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean() # iou loss
# landmark regression
plandmarks = ps[:, 5+model.nc:].sigmoid() * 2. - 0.5
glandmarks = tbox[i][:,4:]
no_landmarks_f = ~(glandmarks == torch.zeros_like(glandmarks[0])).all(1)
plandmarks = plandmarks[no_landmarks_f]
glandmarks = glandmarks[no_landmarks_f]
llandmark += MSElandmarks(plandmarks, glandmarks)
print("llandmark:", llandmark)
# Objectness
tobj[b, a, gj, gi] = (1.0 - model.gr) + model.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
# Classification
if model.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:5+model.nc], cn, device=device) # targets
t[range(n), tcls[i]] = cp
lcls += BCEcls(ps[:, 5:5+model.nc], t) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss
s = 3 / no # output count scaling
lbox *= h['box'] * s
lobj *= h['obj'] * s * (1.4 if no == 4 else 1.)
lcls *= h['cls'] * s
llandmark *= h['box'] * s * 0.1
print(lbox.shape, llandmark.shape)
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls + llandmark
return loss * bs, torch.cat((lbox, lobj, lcls, loss, llandmark)).detach()
def compute_loss_refinenet(p, targets, boxes, model, imgs):
device = targets.device
##########
# try:
# import cv2
# ig = (imgs[0].permute(1,2,0)*255).cpu().numpy().copy()
# lbox = boxes[0][0]
# x1 = int(lbox[0])
# x2 = int(lbox[1])
# x3 = int(lbox[2])
# x4 = int(lbox[3])
# name = x3
# print(x1,x2,x3,x4)
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# cv2.imwrite('yanzheng/'+str(name)+'_1.jpg',ig)
# except:
# pass
all_batch_target = build_targets_forbatch([320, 320], targets, boxes)
indices, tpoint, tcls, tbox = build_targets_forlayer(p, all_batch_target)
###############444 yanzheng
# import cv2
# ig = (imgs[0].permute(1,2,0)*255).cpu().numpy().copy()
# targets = targets[0]*320
# if int(targets[0])==0:
# x1 = int(targets[2])
# x2 = int(targets[3])
# x3 = int(targets[4])
# x4 = int(targets[5])
# name = x3
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# x1 = int(targets[6])
# x2 = int(targets[7])
# x3 = int(targets[8])
# x4 = int(targets[9])
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# cv2.imwrite('yanzheng/'+str(name)+'_2.jpg',ig)
# ##########
# try:
# import cv2
# ig = (imgs[0].permute(1,2,0)*255).cpu().numpy().copy()
# lbox = boxes[0][0]
# x1 = int(lbox[0])
# x2 = int(lbox[1])
# x3 = int(lbox[2])
# x4 = int(lbox[3])
# # name = x3
# print(x1,x2,x3,x4)
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# cv2.imwrite('yanzheng/'+str(name)+'_1.jpg',ig)
# except:
# pass
###############333 yanzheng
# try:
# import cv2
# ig = (imgs[0].permute(1,2,0)*255).cpu().numpy().copy()
# lbox = boxes[0][0]
# abt = all_batch_target[0]
# x1 = int(lbox[0]+(lbox[2]-lbox[0])*abt[2])
# x2 = int(lbox[1]+(lbox[3]-lbox[1])*abt[3])
# # cv2.po
# x3 = int(lbox[0]+(lbox[2]-lbox[0])*abt[4])
# x4 = int(lbox[1]+(lbox[3]-lbox[1])*abt[5])
# name = str(x3)+str(x4)
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# x1 = int(lbox[0]+(lbox[2]-lbox[0])*abt[6])
# x2 = int(lbox[1]+(lbox[3]-lbox[1])*abt[7])
# x3 = int(lbox[0]+(lbox[2]-lbox[0])*abt[8])
# x4 = int(lbox[1]+(lbox[3]-lbox[1])*abt[9])
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# cv2.imwrite('yanzheng/'+str(name)+'.jpg',ig)
# except:
# print('xxxxxxxxxxxx')
bs = p[0][..., 0].shape[0]
# # print(bs)
lcls, lbox, lobj, lpoint_loss = torch.zeros(1, device=device), torch.zeros(
1,
device=device), torch.zeros(1,
device=device), torch.zeros(1,
device=device)
h = model.hyp # hyperparameters
# Define criteria
BCEcls = nn.BCEWithLogitsLoss(
pos_weight=torch.Tensor([h['cls_pw']])).to(device)
BCEobj = nn.BCEWithLogitsLoss(
pos_weight=torch.Tensor([h['obj_pw']])).to(device)
# Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3
cp, cn = smooth_BCE(eps=0.0)
# Focal loss
g = h['fl_gamma'] # focal loss gamma
if g > 0:
BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)
# Losses
nt = 0 # number of targets
no = len(p) # number of outputs
balance = [4.0, 1.0, 0.4] if no == 3 else [4.0, 1.0, 0.4, 0.1
] # P3-5 or P3-6
for i, pi in enumerate(p): # layer index, layer predictions
b, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
# print(n)
if n:
nt += n # cumulative targets
ps = pi[b, gj, gi] # prediction subset corresponding to targets
# Regression
if True:
# print(ps.shape)
# p_fourpoint = (ps[:, :8].sigmoid()-0.5)*2
p_fourpoint = (ps[:, :8].sigmoid() * 2 - 0.5) * 2
#### yanzheng
# try:
# import cv2
# ig = (imgs[0].permute(1,2,0)*255).cpu().numpy().copy()
# lbox__ = boxes[0][0]
# b, gj_, gi_ = indices[0][0][0],indices[0][1][0],indices[0][2][0]
# p_fourpoint___ = p_fourpoint[0]
# x1 = int((p_fourpoint___[0]+gi_)/2*(lbox__[2]-lbox__[0])+lbox__[0])
# x2 = int((p_fourpoint___[1]+gj_)/2*(lbox__[3]-lbox__[1])+lbox__[1])
# x3 = int((p_fourpoint___[2]+gi_)/2*(lbox__[2]-lbox__[0])+lbox__[0])
# x4 = int((p_fourpoint___[3]+gj_)/2*(lbox__[3]-lbox__[1])+lbox__[1])
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# x1 = int((p_fourpoint___[4]+gi_)/2*(lbox__[2]-lbox__[0])+lbox__[0])
# x2 = int((p_fourpoint___[5]+gj_)/2*(lbox__[3]-lbox__[1])+lbox__[1])
# x3 = int((p_fourpoint___[6]+gi_)/2*(lbox__[2]-lbox__[0])+lbox__[0])
# x4 = int((p_fourpoint___[7]+gj_)/2*(lbox__[3]-lbox__[1])+lbox__[1])
# cv2.line(ig,(x1,x2),(x3,x4),(0,255,255),2)
# cv2.imwrite('yanzheng/'+str(name)+'_1.jpg',ig)
# except:
# pass
# print('xxxxxxxxxxxx')
# exit()
# print(p_fourpoint.shape)
# print(tpoint[i].shape)
lpoint_loss += F.mse_loss(p_fourpoint, tpoint[i])
# print(pi.shape)
pbox = get_rec_box_for_predict(p_fourpoint)
# pxy =
# pwh = (ps[:, 2:4].sigmoid()*torch.tensor([2,2]).to(device)).to(device)
# pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box
# iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, DIoU=True,CIoU=True) # iou(prediction, target)
iou = bbox_iou(pbox.T.to(device),
tbox[i].to(device),
x1y1x2y2=False,
CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean()
# Objectness
tobj[b, gj,
gi] = (1.0 -
model.gr) + model.gr * iou.detach().clamp(0).type(
tobj.dtype) # iou ratio
# print(torch.max(tobj),torch.max(pi[..., 4]))
# Classification
if model.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 9:], cn, device=device) # targets
t[range(n), tcls[i]] = cp
lcls += BCEcls(ps[:, 9:], t) # BCE
lobj += BCEobj(pi[..., 8], tobj) * balance[i] # obj loss
s = 3 / no # output count scaling
lbox *= h['box'] * s * 0.5
lobj *= h['obj'] * s * (1.4 if no == 4 else 1.) * 0.5
lcls *= h['cls'] * s * 0.5
bs = tobj.shape[0] # batch size
# else:
loss = lbox + lobj + lcls + lpoint_loss
loss = loss * 0.01
return loss * bs, torch.cat((lbox, lobj, lcls, lpoint_loss, loss)).detach()
# p:N*85*w*h
# target:K*6
# boxes: N*4
return 0
def compute_loss(epoch,p, targets, model): # predictions, targets, model
device = targets.device
loss_anchor = compute_loss_free_anchor(p[3],targets,model)
# if loss_anchor !=0:
# loss_anchor *=0.5
bs = p[0][...,0].shape[0]
# # print(bs)
if epoch<300 and loss_anchor!=0:
return loss_anchor*bs, torch.cat((loss_anchor, loss_anchor, loss_anchor, loss_anchor)).detach()
lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)
tcls, tbox, indices, anchors = build_targets(p, targets, model) # targets
h = model.hyp # hyperparameters
# Define criteria
BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['cls_pw']])).to(device)
BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['obj_pw']])).to(device)
# Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3
cp, cn = smooth_BCE(eps=0.0)
# Focal loss
g = h['fl_gamma'] # focal loss gamma
if g > 0:
BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)
# Losses
nt = 0 # number of targets
no = len(p) # number of outputs
balance = [4.0, 1.0, 0.4] if no == 3 else [4.0, 1.0, 0.4, 0.1] # P3-5 or P3-6
for i, pi in enumerate(p): # layer index, layer predictions
if i ==3:
continue
# pi = pi.repeat()
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
a = torch.zeros(a.shape,device=device).to(torch.int64)
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
# print(n)
if n:
nt += n # cumulative targets
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression
# print(ps.shape)
if True:
panchor = (p[3][i].sigmoid()*2)**3*2
panchor = panchor[b,gj,gi]
# print(panchor.shape)
# skskskskks = ps[:, 2:4]
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = ps[:, 2:4].sigmoid()* panchor
pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box
# iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, DIoU=True,CIoU=True) # iou(prediction, target)
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False,CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean()
# else:
# pxy = ps[:, :2].sigmoid() * 2. - 0.5
# pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]
# pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box
# iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)
# lbox += (1.0 - iou).mean()*2 # iou loss
# Objectness
tobj[b, a, gj, gi] = (1.0 - model.gr) + model.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
# Classification
if model.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:], cn, device=device) # targets
t[range(n), tcls[i]] = cp
lcls += BCEcls(ps[:, 5:], t) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss
s = 3 / no # output count scaling
lbox *= h['box'] * s
lobj *= h['obj'] * s * (1.4 if no == 4 else 1.)
lcls *= h['cls'] * s
bs = tobj.shape[0] # batch size
# print(loss_anchor)
# loss = lbox + lobj + lcls
# if loss_anchor!=0:
# loss = lbox+lobj+lcls+loss_anchor
# loss =loss*1
# return loss * bs, torch.cat((lbox, lobj, loss_anchor, loss)).detach()
# else:
loss = lbox +lobj+lcls
# loss = loss*3
return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()
def __call__(self, p, targets): # predictions, targets, model
device = targets.device
lcls, lbox, lobj, lmark = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)
tcls, tbox, indices, anchors, tlandmarks, lmks_mask = self.build_targets(p, targets) # targets
# Losses
for i, pi in enumerate(p): # layer index, layer predictions
b, a, gj, gi = indices[i] # image, anchor, gridy, gridx
tobj = torch.zeros_like(pi[..., 0], device=device) # target obj
n = b.shape[0] # number of targets
#print('pi: ', pi.shape)
if n:
ps = pi[b, a, gj, gi] # prediction subset corresponding to targets
# Regression
pxy = ps[:, :2].sigmoid() * 2. - 0.5
pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]
pbox = torch.cat((pxy, pwh), 1) # predicted box
iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)
lbox += (1.0 - iou).mean() # iou loss
# Objectness
tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio
#print('tobj: ',tobj[b, a, gj, gi] )
# Classification
if self.nc > 1: # cls loss (only if multiple classes)
t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets
t[range(n), tcls[i]] = self.cp
lcls += self.BCEcls(ps[:, 5:], t) # BCE
# Append targets to text file
# with open('targets.txt', 'a') as file:
# [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)]
#landmarks loss
#print('ps: ', ps.shape)
plandmarks = ps[:,5:13].sigmoid() * 8. - 4.
#print('anchors: ', anchors[i].shape)
#print('plandmarks: ', plandmarks.shape)
plandmarks[:, 0:2] = plandmarks[:, 0:2] * anchors[i]
plandmarks[:, 2:4] = plandmarks[:, 2:4] * anchors[i]
plandmarks[:, 4:6] = plandmarks[:, 4:6] * anchors[i]
plandmarks[:, 6:8] = plandmarks[:, 6:8] * anchors[i]
lmark += self.landmarks_loss(plandmarks, tlandmarks[i], lmks_mask[i])
#print('tobj: ',tobj)
obji = self.BCEobj(pi[..., 4], tobj)
lobj += obji * self.balance[i] # obj loss
if self.autobalance:
self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()
if self.autobalance:
self.balance = [x / self.balance[self.ssi] for x in self.balance]
lbox *= self.hyp['box']
lobj *= self.hyp['obj']
lcls *= self.hyp['cls']
lmark *= self.hyp['landmark']
bs = tobj.shape[0] # batch size
loss = lbox + lobj + lcls + lmark
return loss * bs, torch.cat((lbox, lobj, lcls, lmark, loss)).detach()
def spider_sense(headDet, weapDet, frames, im0, thres, mask, device):
detections = [False, False]
headThres = {2: 0.21, 3: 0.15, 4: 0.09}
# removing head detections that don't meet the necessary width threshold
headDet[-1] = [
det for det in headDet[-1]
if float(2 * (det[2] - det[0]) / im0.shape[1]) >= headThres[thres]
]
# removing weapon detections that are too wide
weapDet[-1] = [
det for det in weapDet[-1]
if float(2 * (det[2] - det[0]) / im0.shape[1]) < 0.8
]
# adding new detections from second last with current if >= 2 detections
if opt.genDet and (len(headDet) >= 2 and len(frames) >= 2):
print("Generating New Detections", len(headDet[-2]), len(weapDet[-2]))
genDet(frames[-2], frames[-1], headDet[-2], mask, device)
genDet(frames[-2], frames[-1], weapDet[-2], mask, device)
print("Done:", len(headDet[-2]), len(weapDet[-2]))
# Doing context check on remaining head and weapons and changing detections if needed
if len(headDet[-1]) > 0 and len(headDet) == opt.filterLen:
for detection in headDet[-1]:
context = 0
tempDet = detection
for i in range(opt.filterLen - 2, -1, -1): # each frame
for j in range(0, len(headDet[i])): # each detection in frame
if bbox_iou(tempDet, headDet[i][j], DIoU=True) >= 0.3:
# if dist_check(tempDet, headDet[i][j]):
context += 1
tempDet = headDet[i][j]
break
if context != (opt.filterLen - i - 1):
break
if context >= opt.filterLen - 2:
detections[0] = True
break
noContext = 0
if len(weapDet[-1]) > 0 and len(weapDet) == opt.filterLen:
for detection in weapDet[-1]:
context = 0
tempDet = detection
for i in range(opt.filterLen - 2, -1, -1): # each frame
for j in range(0, len(weapDet[i])): # each detection in frame
if tempDet[5] == weapDet[i][j][5] and bbox_iou(
tempDet, weapDet[i][j], DIoU=True) >= 0.3:
# if tempDet[5] == weapDet[i][j][5] and dist_check(tempDet, weapDet[i][j]):
context += 1
tempDet = weapDet[i][j]
break
print("Context", context)
print("filterLen - i - 1", opt.filterLen - i - 1)
if context != (opt.filterLen - i - 1):
break
if context >= opt.filterLen - 2:
detections[1] = True
break
return detections