def decode_txn_boxes(self, loc_preds, cls_preds, batchsize, score_thresh):
box_preds = self.decode_loc(loc_preds)
box_preds = opts.batch_to_time(box_preds, batchsize).data
cls_preds = opts.batch_to_time(cls_preds, batchsize).data
tbins, batchsize = box_preds.shape[:2]
box_preds = box_preds.cpu()
cls_preds = cls_preds.cpu()
if self.nms == np_box_nms:
box_preds = box_preds.cpu().numpy()
cls_preds = cls_preds.cpu().numpy()
targets = []
for t in range(tbins):
targets_t = []
for i in range(batchsize):
boxes, labels, scores = self.multiclass_nms(
box_preds[t, i],
cls_preds[t, i],
score_thresh=score_thresh,
nms_thresh=0.6)
targets_t.append((boxes, labels, scores))
targets.append(targets_t)
return targets
def _embeddings_loss(self, emb, ids, cls_targets, batchsize):
"""
In Time, promote similarity between targets with same id
:param pred_embeddings: [TN, #anchors, D]
:param ids: [TN, #nanchors]
:return:
"""
pos = (cls_targets > 0).view(-1)
emb = emb.view(-1, emb.size(-1))[pos]
ids = batch_to_time(ids, batchsize)
offsets = 100 * torch.arange(batchsize)[None, :, None].to(ids.device)
ids = ids + offsets
ids = time_to_batch(ids)[0]
ids = ids.view(-1)[pos]
#TODO:
#1. do this per frame not globally (benchmark this)
#2. weight id with iou loss = iou * id * (1-cos_mat) + (1-id) * max(0, cos_mat - margin)
# We do cos(x1, x2) between every vectors
# When they have same id = we use loss as 1 - cos (penalize for being far)
# When they have not same id = max(0, cos - margin) (penalize for not being close)
y = (ids[:, None] == ids[None, :])
norms = torch.norm(emb, dim=1)
norm_matrix = norms[:, None] * norms[None, :]
dot_matrix = torch.mm(emb, emb.t())
cos_matrix = dot_matrix / norm_matrix
margin = 0.5
loss = torch.where(y, 1 - cos_matrix, F.relu(cos_matrix - margin))
return loss.mean()
def _asso_loss(self, pred_scores, batchsize):
"""
In Time, just promote similar pairwise scores
:param pred_scores:
:param target_scores:
:return:
"""
pred_scores_txn = batch_to_time(pred_scores, batchsize)
loss_asso = pred_scores_txn.sum(dim=-2).std(
dim=0) #sum accross anchors per class
return loss_asso
def upsample(self, x, feat_shape):
x, n = time_to_batch(x)
x = F.interpolate(x, size=feat_shape, mode='nearest')
x = batch_to_time(x, n)
return x
def forward_parallel(self, x):
t, n = x.size(0), x.size(1)
x = time_to_batch(x)[0]
x = self.module(x)
x = batch_to_time(x, n)
return x
def sequence_upsample(x, y):
x, n = time_to_batch(x)
x = F.interpolate(x, size=y.shape[-2:], mode='nearest')
x = batch_to_time(x, n)
return x