def embedding_accuracy(embeddings, labels, perspectives, device=None):
"""Computes the ratio of positive embeddings in the batch which are
closer together than any negative pair.
Args:
embeddings: (N,D) FloatTensor - TCN embeddings of inputs
labels: (N,) FloatTensor - labels indicating positive pairs
perspectives: (N,) FloatTensor - labels indicating video
perspective
device: torch.cuda.device
Returns:
accuracy Tensor(float)
"""
# Tools.pyout(embeddings)
# compute pairwise distance matrix
pdist_matrix = Tools.pdist(embeddings)
# Tools.pyout(pdist_matrix)
# mask labels equal
leq_mask = Tools.pequal(labels).float()
# mask perspectives equal
peq_mask = Tools.pequal(perspectives).float()
# mask positive pairs: same label, different perspective
pos_mask = leq_mask * (1 - peq_mask)
# mask negative pairs: different label, same perspective
neg_mask = (1 - leq_mask) * peq_mask
# compute max distance between positive pairs for each entry
pos_dists, _ = torch.max(
pdist_matrix * pos_mask, dim=1, keepdim=True)
# compute min distance between negative pairs for each entry
neg_dists = torch.where(
neg_mask > 0,
pdist_matrix,
torch.full(neg_mask.size(), float("inf")).to(device))
neg_dists, _ = torch.min(neg_dists, dim=1, keepdim=True)
# compute ratio where positive distance is smaller than all
# negative distance element wise
dist_diff = (pos_dists < neg_dists).float()
# reduce mean
accuracy = sum(dist_diff) / dist_diff.size()[0]
return accuracy.squeeze()
def embedding_accuracy_ratio(embeddings, labels, perspectives):
"""Computes the ratio of positive embeddings in the batch which are
closer together than any negative pair.
Args:
embeddings: (N,D) FloatTensor - TCN embeddings of inputs
labels: (N,) FloatTensor - labels indicating positive pairs
perspectives: (N,) FloatTensor - labels indicating video
perspective
device: torch.cuda.device
Returns:
accuracy Tensor(float)
"""
# compute pairwise distance matrix
pdist_matrix = Tools.pdist(embeddings)
# mask labels equal
leq_mask = Tools.pequal(labels).float()
# mask perspectives equal
peq_mask = Tools.pequal(perspectives).float()
# mask positive pairs: same label, different perspective
pos_mask = leq_mask * (1 - peq_mask)
# mask negative pairs: different label, same perspective
neg_mask = (1 - leq_mask) * peq_mask
# compute max distance between positive pairs for each entry
pos_dists, _ = torch.max(
pdist_matrix * pos_mask, dim=1, keepdim=True)
neg_se_pos = (
pdist_matrix <= pos_dists.expand_as(pdist_matrix)).float()
neg_se_pos = torch.sum(neg_se_pos * neg_mask, dim=1)
neg_se_pos /= torch.sum(neg_mask, dim=1)
neg_se_pos[neg_se_pos != neg_se_pos] = 0. # replace nan by 0.
# reduce mean
accuracy_ratio = torch.mean(neg_se_pos)
return accuracy_ratio
def triplet_semihard_loss(embeddings,
labels,
perspectives,
margin=1.0,
device=None):
"""Computes triplet loss with semi-hard negative mining.
Encourages the distance between positive pairs to be smaller than the
minimum distance between the anchor and all negative embeddings from
the same perspective as the anchor which are further than the positive
distance plus the margin. If all negative embeddings are closer to the
anchor than that, the maximum distance between the anchor and negative
embeddings is used instead.
Args:
embeddings: (N,D) FloatTensor - TCN embeddings of inputs
labels: (N,) FloatTensor - labels indicating positive pairs
perspectives: (N,) FloatTensor - labels indicating video
perspective
margin: float
device: torch.cuda.device
Returns:
triplet loss Tensor(float)
"""
# compute pairwise distance matrix
pdist_matrix = Tools.pdist(embeddings)
# print("\nPDIST MATRIX")
# print(pdist_matrix)
# mask labels equal
leq_mask = Tools.pequal(labels).float()
# mask perspectives equal
peq_mask = Tools.pequal(perspectives).float()
# mask positive pairs: same label, different perspective
pos_mask = leq_mask * (1 - peq_mask)
# mask negative pairs: different label, same perspective
neg_mask = (1 - leq_mask) * peq_mask
# print("\nMASKS")
# print(pos_mask)
# print(neg_mask)
# compute max distance between positive pairs for each entry
pos_dists, _ = torch.max(
pdist_matrix * pos_mask, dim=1, keepdim=True)
# print("\nPOS_DISTS")
# print(pos_dists)
# mask semi-hard negative pairs: all negative pairs of which the
# distance to the anchor is greater than the positive distance and
# smaller than the positive distance plus the margin
neg_ou_mask = torch.ge(
pdist_matrix,
pos_dists.expand_as(pdist_matrix)).float() * \
(1 - torch.ge(
pdist_matrix,
pos_dists.expand_as(pdist_matrix) + margin).float()) * \
neg_mask
# mask hard negative pairs: all negative pairs of which the distance
# to the anchor is smaller than the positive distance
neg_in_mask = neg_mask * (1 - neg_ou_mask) * torch.ge(
pos_dists.expand_as(pdist_matrix) + margin,
pdist_matrix).float()
# compute the minimum distance to the anchor of all negative pairs
# which are outside pos dist + margin for each entry
neg_ou_dists = torch.where(
neg_ou_mask > 0,
pdist_matrix,
torch.full(neg_ou_mask.size(), float("inf")).to(device))
neg_ou_dists = torch.min(
neg_ou_dists, dim=1, keepdim=True)[0]
# print("\nNEG MASKS")
# print(neg_ou_mask * pdist_matrix)
# print(neg_in_mask)
# compute the maximum distance to the anchor of all negative pairs
# which are inside pos_dist + margin for each entry
neg_in_dists = torch.max(
pdist_matrix * neg_in_mask, dim=1, keepdim=True)[0] * \
(1 - torch.sum(neg_ou_mask, 1, keepdim=True).clamp(0., 1.))
# print("\nNEG_DISTS")
# print(neg_ou_dists)
# print(neg_in_dists)
# compute semi-hard sampled negative distance by mergin outside and
# inside negative pairs for each entry: use min outside distance if
# available, o/w use max inside distance
neg_dists = torch.where(
torch.sum(neg_ou_mask, 1, keepdim=True) > 0,
neg_ou_dists,
neg_in_dists)
# element-wise triplet loss
# 0 if no elements in semi-hard or hard range
L_element = (pos_dists + margin - neg_dists).clamp(min=0.) * \
(torch.sum(neg_ou_mask, dim=1, keepdim=True) +
torch.sum(neg_in_mask, dim=1, keepdim=True)).clamp(0., 1.)
# print("\nELEMENT WISE LOSS")
# print(L_element)
# reduce by mean for batch loss
loss = torch.mean(L_element)
return loss
