def forward(self, outputs, batch):
opt = self.opt
hm_loss, wh_loss, off_loss, seg_loss = 0, 0, 0, 0
for s in range(opt.num_stacks):
output = outputs[s]
if not opt.mse_loss:
output['hm'] = _sigmoid(output['hm'])
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
wh_loss += self.crit_reg(output['wh'], batch['reg_mask'],
batch['ind'],
batch['wh']) / opt.num_stacks
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'],
batch['reg']) / opt.num_stacks
seg_loss += self.crit_seg(output['seg'], output['seg_feat'],
batch['ind'], batch['seg'])
loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + \
opt.off_weight * off_loss + opt.seg_weight * seg_loss
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'wh_loss': wh_loss,
'off_loss': off_loss,
'seg_loss': seg_loss
}
return loss, loss_stats
def forward(self, outputs, batch):
"""
:param outputs:
:param batch:
:return:
"""
opt = self.opt
hm_loss, wh_loss, off_loss, id_loss = 0.0, 0.0, 0.0, 0.0 # 初始化4个loss为0
for s in range(opt.num_stacks):
output = outputs[s]
if not opt.mse_loss:
output['hm'] = _sigmoid(output['hm'])
# 计算heatmap loss
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
if opt.wh_weight > 0:
if opt.dense_wh:
mask_weight = batch['dense_wh_mask'].sum() + 1e-4
wh_loss += (self.crit_wh(output['wh'] * batch['dense_wh_mask'],
batch['dense_wh'] * batch['dense_wh_mask']) /
mask_weight) / opt.num_stacks
else: # 计算box尺寸的L1/Smooth L1 loss
wh_loss += self.crit_reg(
output['wh'], batch['reg_mask'],
batch['ind'], batch['wh']) / opt.num_stacks
if opt.reg_offset and opt.off_weight > 0: # 计算box中心坐标偏移的L1 loss
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'], batch['reg']) / opt.num_stacks
# 检测目标id分类的交叉熵损失
if opt.id_weight > 0:
id_head = _tranpose_and_gather_feat(output['id'], batch['ind'])
id_head = id_head[batch['reg_mask'] > 0].contiguous() # 只有有目标的像素才计算id loss
id_head = self.emb_scale * F.normalize(id_head)
id_target = batch['ids'][batch['reg_mask'] > 0] # 有目标的track id
id_output = self.classifier.forward(id_head).contiguous() # 用于检测目标分类的最后一层是FC?
id_loss += self.IDLoss(id_output, id_target)
# id_loss += self.IDLoss(id_output, id_target) + self.TriLoss(id_head, id_target)
# loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + opt.off_weight * off_loss + opt.id_weight * id_loss
det_loss = opt.hm_weight * hm_loss \
+ opt.wh_weight * wh_loss \
+ opt.off_weight * off_loss
loss = torch.exp(-self.s_det) * det_loss \
+ torch.exp(-self.s_id) * id_loss \
+ (self.s_det + self.s_id)
loss *= 0.5
# print(loss, hm_loss, wh_loss, off_loss, id_loss)
loss_stats = {'loss': loss,
'hm_loss': hm_loss,
'wh_loss': wh_loss,
'off_loss': off_loss,
'id_loss': id_loss}
return loss, loss_stats
def forward(self, outputs, batch):
opt = self.opt
hm_loss, wh_loss, off_loss = 0, 0, 0
for s in range(opt.num_stacks):
output = outputs[s]
if not opt.mse_loss:
output['hm'] = _sigmoid(output['hm'])
if opt.eval_oracle_hm:
output['hm'] = batch['hm']
if opt.eval_oracle_wh:
output['wh'] = torch.from_numpy(
gen_oracle_map(batch['wh'].detach().cpu().numpy(),
batch['ind'].detach().cpu().numpy(),
output['wh'].shape[3],
output['wh'].shape[2])).to(opt.device)
if opt.eval_oracle_offset:
output['reg'] = torch.from_numpy(
gen_oracle_map(batch['reg'].detach().cpu().numpy(),
batch['ind'].detach().cpu().numpy(),
output['reg'].shape[3],
output['reg'].shape[2])).to(opt.device)
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
if opt.wh_weight > 0:
if opt.dense_wh:
mask_weight = batch['dense_wh_mask'].sum() + 1e-4
wh_loss += (self.crit_wh(
output['wh'] * batch['dense_wh_mask'],
batch['dense_wh'] * batch['dense_wh_mask']) /
mask_weight) / opt.num_stacks
elif opt.cat_spec_wh:
wh_loss += self.crit_wh(
output['wh'], batch['cat_spec_mask'], batch['ind'],
batch['cat_spec_wh']) / opt.num_stacks
else:
wh_loss += self.crit_reg(output['wh'], batch['reg_mask'],
batch['ind'],
batch['wh']) / opt.num_stacks
if opt.reg_offset and opt.off_weight > 0:
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'],
batch['reg']) / opt.num_stacks
loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + \
opt.off_weight * off_loss
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'wh_loss': wh_loss,
'off_loss': off_loss
}
return loss, loss_stats
def forward(self, outputs, batch):
opt = self.opt
hm_loss, dep_loss, rot_loss, dim_loss = 0, 0, 0, 0
wh_loss, off_loss = 0, 0
for s in range(opt.num_stacks):
output = outputs[s]
output['hm'] = _sigmoid(output['hm'])
output['dep'] = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
if opt.eval_oracle_dep:
output['dep'] = torch.from_numpy(
gen_oracle_map(batch['dep'].detach().cpu().numpy(),
batch['ind'].detach().cpu().numpy(),
opt.output_w, opt.output_h)).to(opt.device)
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
if opt.dep_weight > 0:
dep_loss += self.crit_reg(output['dep'], batch['reg_mask'],
batch['ind'],
batch['dep']) / opt.num_stacks
if opt.dim_weight > 0:
dim_loss += self.crit_reg(output['dim'], batch['reg_mask'],
batch['ind'],
batch['dim']) / opt.num_stacks
if opt.rot_weight > 0:
rot_loss += self.crit_rot(output['rot'], batch['rot_mask'],
batch['ind'], batch['rotbin'],
batch['rotres']) / opt.num_stacks
if opt.reg_bbox and opt.wh_weight > 0:
wh_loss += self.crit_reg(output['wh'], batch['rot_mask'],
batch['ind'],
batch['wh']) / opt.num_stacks
if opt.reg_offset and opt.off_weight > 0:
off_loss += self.crit_reg(output['reg'], batch['rot_mask'],
batch['ind'],
batch['reg']) / opt.num_stacks
loss = opt.hm_weight * hm_loss + opt.dep_weight * dep_loss + \
opt.dim_weight * dim_loss + opt.rot_weight * rot_loss + \
opt.wh_weight * wh_loss + opt.off_weight * off_loss
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'dep_loss': dep_loss,
'dim_loss': dim_loss,
'rot_loss': rot_loss,
'wh_loss': wh_loss,
'off_loss': off_loss
}
return loss, loss_stats
def forward(self, outputs, batch):
opt = self.opt
hm_loss, wh_loss, off_loss, id_loss = 0, 0, 0, 0
for s in range(opt.num_stacks):
output = outputs[s]
if not opt.mse_loss:
output['hm'] = _sigmoid(output['hm'])
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
if opt.wh_weight > 0:
if opt.dense_wh:
mask_weight = batch['dense_wh_mask'].sum() + 1e-4
wh_loss += (
self.crit_wh(output['wh'] * batch['dense_wh_mask'],
batch['dense_wh'] * batch['dense_wh_mask']) /
mask_weight) / opt.num_stacks
else:
wh_loss += self.crit_reg(
output['wh'], batch['reg_mask'],
batch['ind'], batch['wh']) / opt.num_stacks
if opt.reg_offset and opt.off_weight > 0:
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'], batch['reg']) / opt.num_stacks
if opt.id_weight > 0:
id_head = _tranpose_and_gather_feat(output['id'], batch['ind'])
id_head = id_head[batch['reg_mask'] > 0].contiguous()
id_head = self.emb_scale * F.normalize(id_head)
id_target = batch['ids'][batch['reg_mask'] > 0]
id_output = self.classifier(id_head).contiguous()
id_loss += self.IDLoss(id_output, id_target)
# id_loss += self.IDLoss(id_output, id_target) + self.TriLoss(id_head, id_target)
# loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + opt.off_weight * off_loss + opt.id_weight * id_loss
det_loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + opt.off_weight * off_loss
loss = torch.exp(-self.s_det) * det_loss + torch.exp(-self.s_id) * id_loss + (self.s_det + self.s_id)
loss *= 0.5
# print(loss, hm_loss, wh_loss, off_loss, id_loss)
loss_stats = {'loss': loss, 'hm_loss': hm_loss,
'wh_loss': wh_loss, 'off_loss': off_loss, 'id_loss': id_loss}
return loss, loss_stats
def _neg_seg_loss(pred, gt, mask):
''' Modified focal loss. Exactly the same as CornerNet.
Runs faster and costs a little bit more memory
Arguments:
pred (batch x c x h x w)
gt_regr (batch x c x h x w)
'''
pred = _sigmoid(pred)
pos_inds = gt.eq(1).float()
neg_inds = gt.lt(1).float()
mask = mask.unsqueeze(-1).expand(pred.size()).float()
pos_loss = torch.log(pred) * pos_inds
neg_loss = torch.log(1 - pred) * neg_inds * mask
num_sample = mask.sum()
pos_loss = pos_loss.sum()
neg_loss = neg_loss.sum()
loss = 0 - (pos_loss * 1.5 + neg_loss) / num_sample
return loss
def forward(self, outputs, batch):
"""
:param outputs:
:param batch:
:return:
"""
opt = self.opt
# 初始化4个loss为0
hm_loss, wh_loss, off_loss, reid_loss = 0.0, 0.0, 0.0, 0.0
for s in range(opt.num_stacks):
# ----- Detection loss
output = outputs[s]
if not opt.mse_loss:
output['hm'] = _sigmoid(output['hm'])
# --- heat-map loss
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
# --- box width and height loss
if opt.wh_weight > 0:
if opt.dense_wh:
mask_weight = batch['dense_wh_mask'].sum() + 1e-4
wh_loss += (self.crit_wh(output['wh'] * batch['dense_wh_mask'],
batch['dense_wh'] * batch['dense_wh_mask']) / mask_weight) \
/ opt.num_stacks
else: # box width and height using L1/Smooth L1 loss
wh_loss += self.crit_reg(output['wh'], batch['reg_mask'],
batch['ind'],
batch['wh']) / opt.num_stacks
# --- bbox center offset loss
if opt.reg_offset and opt.off_weight > 0: # offset using L1 loss
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'],
batch['reg']) / opt.num_stacks
# ----- ReID loss: only process the class requiring ReID
if opt.id_weight > 0: # if ReID is needed
cls_id_map = batch['cls_id_map']
# 遍历每一个需要ReID的检测类别, 计算ReID损失
for cls_id, id_num in self.nID_dict.items():
inds = torch.where(cls_id_map == cls_id)
if inds[0].shape[0] == 0:
# print('skip class id', cls_id)
continue
# --- 取cls_id对应索引处的特征向量
cls_id_head = output['id'][inds[0], :, inds[2], inds[3]]
cls_id_head = self.emb_scale_dict[cls_id] * F.normalize(
cls_id_head) # n × emb_dim
# --- 获取target类别
cls_id_target = batch['cls_tr_ids'][inds[0], cls_id,
inds[2], inds[3]]
# ---分类结果
# 使用普通的全连接层
cls_id_output = self.classifiers[str(cls_id)].forward(
cls_id_head).contiguous()
# 使用Arc margin全连接层
# cls_id_output = self.classifiers[str(cls_id)].forward(cls_id_head, cls_id_target).contiguous()
# --- 累加每一个检测类别的ReID loss
# 选择一: 使用交叉熵优化ReID
reid_loss += self.IDLoss(cls_id_output, cls_id_target)
# 选择二: 使用Circle loss优化ReID
# reid_loss += self.circle_loss(*convert_label_to_similarity(cls_id_output, cls_id_target))
# 选择三: 使用triplet loss优化ReID
# reid_loss += self.IDLoss(cls_id_output, cls_id_target) + self.TriLoss(cls_id_head, cls_id_target)
# loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + opt.off_weight * off_loss + opt.id_weight * id_loss
det_loss = opt.hm_weight * hm_loss \
+ opt.wh_weight * wh_loss \
+ opt.off_weight * off_loss
if opt.id_weight > 0:
loss = torch.exp(-self.s_det) * det_loss \
+ torch.exp(-self.s_id) * reid_loss \
+ (self.s_det + self.s_id)
else:
loss = torch.exp(-self.s_det) * det_loss \
+ self.s_det
loss *= 0.5
# print(loss, hm_loss, wh_loss, off_loss, id_loss)
if opt.id_weight > 0:
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'wh_loss': wh_loss,
'off_loss': off_loss,
'id_loss': reid_loss
}
else:
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'wh_loss': wh_loss,
'off_loss': off_loss
} # only exists det loss
return loss, loss_stats
def forward(self, outputs, batch):
"""
:param outputs:
:param batch:
:return:
"""
opt = self.opt
# 初始化4个loss为0
hm_loss, wh_loss, off_loss, reid_loss = 0.0, 0.0, 0.0, 0.0
for s in range(opt.num_stacks):
output = outputs[s]
if not opt.mse_loss:
output['hm'] = _sigmoid(output['hm'])
# 计算heat-map loss
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
if opt.wh_weight > 0:
if opt.dense_wh:
mask_weight = batch['dense_wh_mask'].sum() + 1e-4
wh_loss += (self.crit_wh(
output['wh'] * batch['dense_wh_mask'],
batch['dense_wh'] * batch['dense_wh_mask']) /
mask_weight) / opt.num_stacks
else: # 计算box尺寸的L1/Smooth L1 loss
wh_loss += self.crit_reg(output['wh'], batch['reg_mask'],
batch['ind'],
batch['wh']) / opt.num_stacks
if opt.reg_offset and opt.off_weight > 0: # 计算box中心坐标偏移的L1 loss
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'],
batch['reg']) / opt.num_stacks
cls_id_map = batch['cls_id_map']
# ----- ReID损失: 仅仅处理需要ReID的类别
if opt.id_weight > 0: # 如果需要训练ReID
# 遍历每一个需要ReID的检测类别, 计算ReID损失
for cls_id, id_num in self.nID_dict.items():
inds = torch.where(cls_id_map == cls_id)
if inds[0].shape[0] == 0:
# print('skip class id', cls_id)
continue
# --- 取cls_id对应索引处的特征向量
cls_id_head = output['id'][inds[0], :, inds[2], inds[3]]
cls_id_head = self.emb_scale_dict[cls_id] * F.normalize(
cls_id_head) # n × emb_dim
# --- 获取target类别
cls_id_target = batch['cls_tr_ids'][inds[0], cls_id,
inds[2], inds[3]]
# ---分类结果
cls_id_output = self.classifiers[str(cls_id)].forward(
cls_id_head).contiguous() # FC layer
# 使用Arc margin FC layer
# cls_id_output = self.classifiers[str(cls_id)].forward(cls_id_head, cls_id_target).contiguous()
# --- 累加每一个检测类别的ReID loss
# 使用交叉熵优化ReID
# reid_loss += self.IDLoss(cls_id_output, cls_id_target)
# 使用Circle loss优化ReID
reid_loss += self.circle_loss(*convert_label_to_similarity(
cls_id_output, cls_id_target))
# 使用triplet loss优化ReID
# reid_loss += self.IDLoss(id_output, id_target) + self.TriLoss(id_head, id_target)
# loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + opt.off_weight * off_loss + opt.id_weight * id_loss
det_loss = opt.hm_weight * hm_loss \
+ opt.wh_weight * wh_loss \
+ opt.off_weight * off_loss
if opt.id_weight > 0:
loss = torch.exp(-self.s_det) * det_loss \
+ torch.exp(-self.s_id) * reid_loss \
+ (self.s_det + self.s_id)
else:
loss = torch.exp(-self.s_det) * det_loss \
+ (self.s_det + self.s_id)
loss *= 0.5
# print(loss, hm_loss, wh_loss, off_loss, id_loss)
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'wh_loss': wh_loss,
'off_loss': off_loss,
'id_loss': reid_loss
}
return loss, loss_stats
def loss(self, stuff, cur_labels, ref_labels):
affs, sims, qlts, sfeats, cur_feats = stuff
# print(cur_labels.shape, ref_labels.shape)
losses = {}
# print(cur_labels[:10])
# print(ref_labels[:20])
sim_target = cur_labels.view(-1, 1) == ref_labels.view(1, -1)
# print(sim_target.shape)
# print(sims[0].shape)
weight_1 = sim_target.new_zeros(sim_target.shape).int()
weight_2 = sim_target.new_zeros(sim_target.shape).int()
pos_cur = torch.nonzero(cur_labels > 0)
pos_ref = torch.nonzero(ref_labels > 0)
weight_1[pos_cur, :] = 1
weight_2[:, pos_ref] = 1
weight = (weight_1 + weight_2).flatten()
pos_pos_inds = torch.nonzero(weight == 2)
pos_neg_inds = torch.nonzero(weight == 1)
neg_neg_inds = torch.nonzero(weight == 0)
pos_pos_cnt = pos_pos_inds.size(0)
pos_neg_cnt = pos_pos_cnt * 2
neg_neg_cnt = pos_pos_cnt * 1
rand_inds = torch.randperm(pos_neg_inds.size(0))
rand_inds = pos_neg_inds[rand_inds[:pos_neg_cnt]]
weight[rand_inds] = 2
rand_inds = torch.randperm(neg_neg_inds.size(0))
rand_inds = neg_neg_inds[rand_inds[:neg_neg_cnt]]
weight[rand_inds] = 2
# weight[:10, :10] = 1
# weight[pos_cur, :10] = 1
# weight[:10, pos_ref] = 1
sim_inds = torch.nonzero(weight == 2)
# print(sim_inds.shape)
# print(sim_inds.shape, pos_cur.numel(), pos_ref.numel())
sim_target = sim_target.reshape(-1).float()
# print(sim_target.shape)
# print(sim_inds.shape, weight.shape, sim_target.shape)
sim_target = sim_target[sim_inds]
# print(sim_inds[:10], sim_target.shape)
# print(sims[0].flatten()[:10], sim_target[:10])
# print(torch.nonzero(sim_target > 0).size(0), torch.nonzero(sim_target == 0).size(0), pos_pos_cnt, weight.numel())
# print(self.rel.stages)
for i in range(self.rel.stages):
sim = sims[i].sum(dim=1).reshape(-1)[sim_inds]
sim = _sigmoid(sim)
qlt = _sigmoid(qlts[i])
sfeat = sfeats[i]
# print(sfeat.device)
# print(self.pre_cls[0].weight.device)
pre_cls_pred = _sigmoid(self.pre_cls(sfeat))
# print(pre_cls_pred[:10])
# print(cls_target[:10])
losses['pre_cls.%d.loss' % i] = self.binary_loss(
pre_cls_pred.flatten(),
cur_labels.flatten().float()) / self.rel.stages * 0.5
qlt_target = (1 - torch.abs(cur_labels.flatten().float() -
pre_cls_pred.flatten())).detach()
# print(qlt_target[:10])
losses['qlt.%d.loss' % i] = self.binary_loss(
qlt.flatten(), qlt_target.flatten()) / self.rel.stages * 0.5
losses['sim.%d.loss' % i] = self.binary_loss(
sim.flatten(), sim_target.flatten()) / self.rel.stages
# print(losses['sim.%d.loss' % 0])
# input()
loss = 0
for v in losses.values():
loss += v
loss = loss * 0.5
# losses['loss_rel'] = loss
# for k in list(losses.keys()):
# losses[k] = float(losses[k])
return loss, losses
def forward(self, outputs, batch):
opt = self.opt
hm_loss, wh_loss, off_loss = 0, 0, 0
hp_loss, off_loss, hm_hp_loss, hp_offset_loss = 0, 0, 0, 0
for s in range(opt.num_stacks):
output = outputs[s]
output['hm'] = _sigmoid(output['hm'])
if opt.hm_hp and not opt.mse_loss:
output['hm_hp'] = _sigmoid(output['hm_hp'])
if opt.eval_oracle_hmhp:
output['hm_hp'] = batch['hm_hp']
if opt.eval_oracle_hm:
output['hm'] = batch['hm']
if opt.eval_oracle_kps:
if opt.dense_hp:
output['hps'] = batch['dense_hps']
else:
output['hps'] = torch.from_numpy(
gen_oracle_map(batch['hps'].detach().cpu().numpy(),
batch['ind'].detach().cpu().numpy(),
opt.output_res,
opt.output_res)).to(opt.device)
if opt.eval_oracle_hp_offset:
output['hp_offset'] = torch.from_numpy(
gen_oracle_map(batch['hp_offset'].detach().cpu().numpy(),
batch['hp_ind'].detach().cpu().numpy(),
opt.output_res,
opt.output_res)).to(opt.device)
hm_loss += self.crit(output['hm'], batch['hm']) / opt.num_stacks
if opt.dense_hp:
mask_weight = batch['dense_hps_mask'].sum() + 1e-4
hp_loss += (self.crit_kp(
output['hps'] * batch['dense_hps_mask'], batch['dense_hps']
* batch['dense_hps_mask']) / mask_weight) / opt.num_stacks
else:
hp_loss += self.crit_kp(output['hps'], batch['hps_mask'],
batch['ind'],
batch['hps']) / opt.num_stacks
if opt.wh_weight > 0:
wh_loss += self.crit_reg(output['wh'], batch['reg_mask'],
batch['ind'],
batch['wh']) / opt.num_stacks
if opt.reg_offset and opt.off_weight > 0:
off_loss += self.crit_reg(output['reg'], batch['reg_mask'],
batch['ind'],
batch['reg']) / opt.num_stacks
if opt.reg_hp_offset and opt.off_weight > 0:
hp_offset_loss += self.crit_reg(
output['hp_offset'], batch['hp_mask'], batch['hp_ind'],
batch['hp_offset']) / opt.num_stacks
if opt.hm_hp and opt.hm_hp_weight > 0:
hm_hp_loss += self.crit_hm_hp(output['hm_hp'],
batch['hm_hp']) / opt.num_stacks
loss = opt.hm_weight * hm_loss + opt.wh_weight * wh_loss + \
opt.off_weight * off_loss + opt.hp_weight * hp_loss + \
opt.hm_hp_weight * hm_hp_loss + opt.off_weight * hp_offset_loss
loss_stats = {
'loss': loss,
'hm_loss': hm_loss,
'hp_loss': hp_loss,
'hm_hp_loss': hm_hp_loss,
'hp_offset_loss': hp_offset_loss,
'wh_loss': wh_loss,
'off_loss': off_loss
}
return loss, loss_stats
