def validation_step(self, batch, batch_idx):
x, y = batch
y_hat, _ = self(x)
loss = F.cross_entropy(y_hat, y)
top1, top5 = calc_topk_accuracy(y_hat, y, (1, 5))
tensorboard_logs = {'val_loss': loss, 'top_1': top1, 'top_5': top5}
return {'val_loss': loss, 'log': tensorboard_logs}
def train_one_epoch(data_loader, model, criterion, optimizer, transforms_cuda,
epoch, args, logger):
model.train()
def tr(x):
B = x.size(0)
return transforms_cuda(x).view(B, 3, args.num_seq, args.seq_len,
args.img_dim,
args.img_dim).transpose(1,
2).contiguous()
for idx, input_seq in tqdm(enumerate(data_loader),
total=len(data_loader),
disable=True):
B = input_seq.size(0)
input_seq = tr(input_seq.cuda())
output, target = model(input_seq)
loss = criterion(output, target)
top1, top5 = calc_topk_accuracy(output, target, (1, 5))
# top1_meter.update(top1.item(), B)
# top5_meter.update(top5.item(), B)
# losses.update(loss.item(), B)
optimizer.zero_grad()
loss.backward()
optimizer.step()
logger.add_scalar('Loss/train', loss.item(), args.iteration)
args.iteration += 1
def summarize_probability(prob_dict, action_to_idx, title):
acc = [AverageMeter(), AverageMeter()]
stat = {}
for vname, item in tqdm(prob_dict.items(), total=len(prob_dict)):
try:
action_name = vname.split('/')[-3]
except:
action_name = vname.split('/')[-2]
target = action_to_idx(action_name)
mean_prob = torch.stack(item['mean_prob'], 0).mean(0)
mean_top1, mean_top5 = calc_topk_accuracy(
mean_prob,
torch.LongTensor([target]).cuda(), (1, 5))
stat[vname] = {'mean_prob': mean_prob.tolist()}
acc[0].update(mean_top1.item(), 1)
acc[1].update(mean_top5.item(), 1)
print('Mean: Acc@1: {acc[0].avg:.4f} Acc@5: {acc[1].avg:.4f}'.format(
acc=acc))
with open(
os.path.join(
os.path.dirname(args.test),
'%s-prob-%s.json' % (os.path.basename(args.test), title)),
'w') as fp:
json.dump(stat, fp)
return acc
def get_loss(self, pred, gt, B, SL, last_size, feature_size, kernel=1):
# pred: B,C,N,H,W
# GT: C,B,N,H*H
score = torch.matmul(
pred.permute(0, 2, 3, 4,
1).contiguous().view(B * SL * last_size**2,
feature_size),
gt.contiguous().view(feature_size, B * SL * last_size**2))
if SL not in self.tgt_dict:
self.tgt_dict[SL] = torch.arange(B * SL * last_size**2)
tgt = self.tgt_dict[SL].to(score.device)
loss = self.ce_loss(score, tgt)
top1, top5 = calc_topk_accuracy(score, tgt, (1, 5))
return loss, top1, top5
def compute_selfsupervised_loss(args, pred, feature_dist, target, sizes_pred,
sizes_mask, B, indices, trainer):
score = compute_scores(args, pred, feature_dist, sizes_pred, B, indices,
trainer)
_, B2, NS, NP, SQ = sizes_mask
# score is a 6d tensor: [B, P, SQ, B2, N, SQ]
# similarity matrix is computed inside each gpu, thus here B == num_gpu * B2
score_flattened = score.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.float().argmax(dim=1)
loss = torch.nn.functional.cross_entropy(score_flattened, target_flattened)
top1, top3, top5 = utils.calc_topk_accuracy(score_flattened,
target_flattened, (1, 3, 5))
results = top1, top3, top5, loss.item(), B
if args.hyp_cone: # Add on top of the previous one, instead of replacing it
dist_fn = PairwiseHypConeDist(K=0.1, fp64_hyper=args.fp64_hyper)
score_cone = dist_fn(pred, feature_dist)
score_cone[score_cone < 0] = 0
pos = score_cone.diag().clone()
neg = torch.relu(args.margin - score_cone)
neg.fill_diagonal_(0)
loss_pos = pos.sum()
loss_neg = neg.sum() / score_cone.shape[0]
loss_cone = loss_pos + loss_neg
_, size_1 = score_cone.shape
score = score_cone[:size_1, :]
pos = score.diagonal(dim1=-2, dim2=-1)
pos_acc = float(
(pos == 0).sum().item()) / float(pos.flatten().shape[0])
k = score.shape[0]
score.as_strided([k], [k + 1]).copy_(torch.zeros(k))
neg_acc = float((score == 0).sum().item() - k) / float(k**2 - k)
loss_cone = loss_cone / (2 * k)
loss = loss_cone + loss
results = pos_acc, neg_acc, loss_cone.item(
), B # These are the reported values
return results, loss
def validate(data_loader, model, criterion, transforms_cuda, device, epoch,
args):
batch_time = AverageMeter()
losses = AverageMeter()
top1_meter = AverageMeter('acc@1', ':.4f')
top5_meter = AverageMeter('acc@5', ':.4f')
model.eval()
def tr(x):
B = x.size(0)
return transforms_cuda(x).view(B,3,args.num_seq,args.seq_len,args.img_dim,args.img_dim)\
.transpose(1,2).contiguous()
with torch.no_grad():
end = time.time()
for idx, (input_seq, target) in tqdm(enumerate(data_loader),
total=len(data_loader)):
B = input_seq.size(0)
input_seq = tr(input_seq.to(device, non_blocking=True))
target = target.to(device, non_blocking=True)
logit, _ = model(input_seq)
loss = criterion(logit, target)
top1, top5 = calc_topk_accuracy(logit, target, (1, 5))
losses.update(loss.item(), B)
top1_meter.update(top1.item(), B)
top5_meter.update(top5.item(), B)
batch_time.update(time.time() - end)
end = time.time()
print('Epoch: [{0}]\t'
'Loss: {loss.avg:.4f} Acc@1: {top1.avg:.4f} Acc@5: {top5.avg:.4f}\t'.
format(epoch, loss=losses, top1=top1_meter, top5=top5_meter))
args.val_plotter.add_data('global/loss', losses.avg, epoch)
args.val_plotter.add_data('global/top1', top1_meter.avg, epoch)
args.val_plotter.add_data('global/top5', top5_meter.avg, epoch)
args.logger.log(
'val Epoch: [{0}]\t'
'Loss: {loss.avg:.4f} Acc@1: {top1.avg:.4f} Acc@5: {top5.avg:.4f}\t'.
format(epoch, loss=losses, top1=top1_meter, top5=top5_meter))
return losses.avg, top1_meter.avg
def compute_selfsupervised_loss(args, pred, feature_dist, target, sizes_pred,
sizes_mask, B):
score = compute_scores(args, pred, feature_dist, sizes_pred, B)
_, B2, NS, NP, SQ = sizes_mask
# score is a 6d tensor: [B, P, SQ, B2, N, SQ]
# similarity matrix is computed inside each gpu, thus here B == num_gpu * B2
score_flattened = score.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.float().argmax(dim=1)
loss = torch.nn.functional.cross_entropy(score_flattened, target_flattened)
top1, top3, top5 = utils.calc_topk_accuracy(score_flattened,
target_flattened, (1, 3, 5))
results = top1, top3, top5, loss.item(), B
return results, loss
def summarize_multiprobability(prob_dict, title):
mAP = AverageMeter()
stat = {}
probs = []
targets = []
for vname, item in tqdm(prob_dict.items(), total=len(prob_dict)):
mean_prob, target = zip(*item)
mean_prob = torch.stack(mean_prob, 0).mean(0)
target = target[0]
probs.append(mean_prob)
targets.append(target)
mean_ap = calc_topk_accuracy(torch.cat(probs), torch.cat(targets)).item()
# stat[vname] = {'mean_prob': mean_prob.tolist()}
mAP.update(mean_ap, 1)
print('Mean: mAP: {mAP.avg:.4f}'.format(mAP=mAP))
with open(
os.path.join(
os.path.dirname(args.test),
'%s-prob-%s.json' % (os.path.basename(args.test), title)),
'w') as fp:
json.dump(stat, fp)
return mAP
def train_one_epoch(data_loader, model, criterion, optimizer, transforms_cuda, epoch, args):
batch_time = AverageMeter('Time',':.2f')
data_time = AverageMeter('Data',':.2f')
losses = AverageMeter('Loss',':.4f')
top1_meter = AverageMeter('acc@1', ':.4f')
top5_meter = AverageMeter('acc@5', ':.4f')
top1_self_meter = AverageMeter('Self-acc@1', ':.4f')
top5_self_meter = AverageMeter('Self-acc@5', ':.4f')
sacc_meter = AverageMeter('Sampling-Acc@%d' % args.topk, ':.2f')
progress = ProgressMeter(
len(data_loader),
[batch_time, data_time, losses, top1_meter, top5_meter, top1_self_meter, top5_self_meter, sacc_meter],
prefix='Epoch:[{}]'.format(epoch))
model.train()
model.module.sampler.eval() # the sampler is always fixed
def tr(x):
B = x.size(0)
return transforms_cuda(x).view(B,3,args.num_seq,args.seq_len,args.img_dim,args.img_dim).transpose(1,2).contiguous()
tic = time.time()
end = time.time()
for idx, (input_seq, vname, _) in enumerate(data_loader):
data_time.update(time.time() - end)
B = input_seq[0].size(0)
input_seq = [tr(i.cuda(non_blocking=True)) for i in input_seq]
vname = vname.cuda(non_blocking=True)
output, mask = model(*input_seq, vname)
mask_sum = mask.sum(1)
loss = multi_nce_loss(output, mask)
top1, top5 = calc_mask_accuracy(output, mask, (1,5))
top1_self, top5_self = calc_topk_accuracy(output, torch.zeros(B, dtype=torch.long).cuda(), (1,5))
del output
losses.update(loss.item(), B)
top1_meter.update(top1.item(), B)
top5_meter.update(top5.item(), B)
top1_self_meter.update(top1_self.item(), B)
top5_self_meter.update(top5_self.item(), B)
if model.module.queue_is_full:
optimizer.zero_grad()
loss.backward()
optimizer.step()
del loss
torch.cuda.empty_cache()
batch_time.update(time.time() - end)
end = time.time()
if idx % args.print_freq == 0:
progress.display(idx)
if args.print:
args.train_plotter.add_data('local/loss', losses.local_avg, args.iteration)
args.train_plotter.add_data('local/top1', top1_meter.local_avg, args.iteration)
args.train_plotter.add_data('local/top5', top5_meter.local_avg, args.iteration)
args.train_plotter.add_data('local/self-top1', top1_self_meter.local_avg, args.iteration)
args.train_plotter.add_data('local/self-top5', top5_self_meter.local_avg, args.iteration)
args.iteration += 1
print('({gpu:1d})Epoch: [{0}][{1}/{2}]\t'
'T-epoch:{t:.2f}\t'.format(epoch, idx, len(data_loader), gpu=args.rank, t=time.time()-tic))
if args.print:
args.train_plotter.add_data('global/loss', losses.avg, epoch)
args.train_plotter.add_data('global/top1', top1_meter.avg, epoch)
args.train_plotter.add_data('global/top5', top5_meter.avg, epoch)
args.train_plotter.add_data('global/self-top1', top1_self_meter.avg, epoch)
args.train_plotter.add_data('global/self-top5', top5_self_meter.avg, epoch)
return losses.avg, top1_meter.avg
def train_one_epoch(data_loader, model, criterion, optimizer, transforms_cuda,
device, epoch, args):
batch_time = AverageMeter('Time', ':.2f')
data_time = AverageMeter('Data', ':.2f')
losses = AverageMeter('Loss', ':.4f')
top1_meter = AverageMeter('acc@1', ':.4f')
top5_meter = AverageMeter('acc@5', ':.4f')
progress = ProgressMeter(
len(data_loader),
[batch_time, data_time, losses, top1_meter, top5_meter],
prefix='Epoch:[{}]'.format(epoch))
if args.train_what == 'last':
model.eval() # totally freeze BN in backbone
else:
model.train()
if args.final_bn:
model.module.final_bn.train()
end = time.time()
tic = time.time()
def tr(x): # transformation on tensor
B = x.size(0)
return transforms_cuda(x).view(B,3,args.num_seq,args.seq_len,args.img_dim,args.img_dim)\
.transpose(1,2).contiguous()
for idx, (input_seq, target) in enumerate(data_loader):
data_time.update(time.time() - end)
B = input_seq.size(0)
input_seq = tr(input_seq.to(device, non_blocking=True))
target = target.to(device, non_blocking=True)
logit, _ = model(input_seq)
loss = criterion(logit, target)
top1, top5 = calc_topk_accuracy(logit, target, (1, 5))
losses.update(loss.item(), B)
top1_meter.update(top1.item(), B)
top5_meter.update(top5.item(), B)
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if idx % args.print_freq == 0:
progress.display(idx)
args.train_plotter.add_data('local/loss', losses.local_avg,
args.iteration)
args.train_plotter.add_data('local/top1', top1_meter.local_avg,
args.iteration)
args.train_plotter.add_data('local/top5', top5_meter.local_avg,
args.iteration)
args.iteration += 1
print('Epoch: [{0}][{1}/{2}]\t'
'T-epoch:{t:.2f}\t'.format(epoch,
idx,
len(data_loader),
t=time.time() - tic))
args.train_plotter.add_data('global/loss', losses.avg, epoch)
args.train_plotter.add_data('global/top1', top1_meter.avg, epoch)
args.train_plotter.add_data('global/top5', top5_meter.avg, epoch)
args.logger.log('train Epoch: [{0}][{1}/{2}]\t'
'T-epoch:{t:.2f}\t'.format(epoch,
idx,
len(data_loader),
t=time.time() - tic))
return losses.avg, top1_meter.avg
def train_one_epoch(data_loader, model, criterion, optimizer, lr_scheduler, transforms_cuda, epoch, args):
batch_time = AverageMeter('Time',':.2f')
data_time = AverageMeter('Data',':.2f')
losses = AverageMeter('Loss',':.4f')
top1_meter = AverageMeter('acc@1', ':.4f')
top5_meter = AverageMeter('acc@5', ':.4f')
progress = ProgressMeter(
len(data_loader),
[batch_time, data_time, losses, top1_meter, top5_meter],
prefix='Epoch:[{}]'.format(epoch))
model.train()
def tr(x):
B = x.size(0)
return transforms_cuda(x).view(B,3,args.num_seq,args.seq_len,args.img_dim,args.img_dim)\
.transpose(1,2).contiguous()
end = time.time()
for idx, input_seq in tqdm(enumerate(data_loader), total=len(data_loader)):
data_time.update(time.time() - end)
B = input_seq.size(0)
input_seq = tr(input_seq.cuda(non_blocking=True))
output, target = model(input_seq)
if args.model == 'infonce': # 'target' is the index of self
loss = criterion(output, target)
top1, top5 = calc_topk_accuracy(output, target, (1,5))
if args.model == 'ubernce': # 'target' is the binary mask
# optimize all positive pairs, compute the mean for num_pos and for batch_size
loss = - (F.log_softmax(output, dim=1) * target).sum(1) / target.sum(1)
loss = loss.mean()
top1, top5 = calc_mask_accuracy(output, target, (1,5))
top1_meter.update(top1.item(), B)
top5_meter.update(top5.item(), B)
losses.update(loss.item(), B)
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
progress.display(idx)
if idx % args.print_freq == 0:
if args.print:
args.train_plotter.add_data('local/loss', losses.local_avg, args.iteration)
args.train_plotter.add_data('local/top1', top1_meter.local_avg, args.iteration)
args.iteration += 1
print('({gpu:1d})Epoch: [{0}][{1}/{2}]\t'
'T-epoch:{t:.2f}\t'.format(epoch, idx, len(data_loader), gpu=args.rank, t=time.time()-tic))
if args.print:
args.train_plotter.add_data('global/loss', losses.avg, epoch)
args.train_plotter.add_data('global/top1', top1_meter.avg, epoch)
args.train_plotter.add_data('global/top5', top5_meter.avg, epoch)
return losses.avg, top1_meter.avg
