def validate(data_loader, model, epoch):
losses = AverageMeter()
accuracy = AverageMeter()
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.eval()
with torch.no_grad():
for idx, input_seq in tqdm(enumerate(data_loader),
total=len(data_loader)):
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[score_, mask_] = model(input_seq)
del input_seq
if idx == 0:
target_, (_, B2, NS, NP, SQ) = process_output(mask_)
# [B, P, SQ, B, N, SQ]
score_flattened = score_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
if args.loss_function == 'CE':
target_flattened = target_flattened.double()
target_flattened = target_flattened.argmax(dim=1)
loss = criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(score_flattened,
target_flattened,
(1, 3, 5))
elif args.loss_function == 'MSE':
target_flattened = target_flattened.float()
loss = criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(
score_flattened, target_flattened.argmax(dim=1), (1, 3, 5))
losses.update(loss.item(), B)
accuracy.update(top1.item(), B)
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
print('[{0}/{1}] Loss {loss.local_avg:.4f}\t'
'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f} \t'.format(
epoch, args.epochs, *[i.avg for i in accuracy_list],
loss=losses))
return losses.local_avg, accuracy.local_avg, [
i.local_avg for i in accuracy_list
]
def ensemble(prob_imgs=None, prob_flow=None, prob_seg=None, prob_kphm=None):
acc_top1 = AverageMeter()
acc_top5 = AverageMeter()
probs = [prob_imgs, prob_flow, prob_seg, prob_kphm]
for idx in range(len(probs)):
if probs[idx] is not None:
probs[idx] = {k[0][0].data: v for k, v in probs[idx].items()}
valid_probs = [x for x in probs if x is not None]
weights = [2, 2, 1, 1]
ovr_probs = {}
for k in valid_probs[0].keys():
ovr_probs[k] = valid_probs[0][k]['prob'] * 0.0
total = 0
for idx in range(len(probs)):
p = probs[idx]
if p is not None:
total += weights[idx]
ovr_probs[k] += p[k]['prob'] * weights[idx]
ovr_probs[k] /= total
top1, top5 = calc_topk_accuracy(ovr_probs[k],
valid_probs[0][k]['target'], (1, 5))
acc_top1.update(top1.item(), 1)
acc_top5.update(top5.item(), 1)
print('Test - Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'.format(
top1=acc_top1, top5=acc_top5))
def valid_step(self, idx, input_seq, losses, accuracy, accuracy_list):
self.model.eval()
self.criterion.eval()
input_seq = self._prepare_sample(input_seq)
B = input_seq.size(0)
[score_, mask_] = self.model(input_seq)
del input_seq
if idx == 0:
target_, (_, self.B2, NS, NP, SQ) = process_output(mask_)
# [B, P, SQ, B, N, SQ]
score_flattened = score_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.argmax(dim=1)
loss = self.criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(score_flattened,
target_flattened, (1, 3, 5))
losses.update(loss.item(), B)
accuracy.update(top1.item(), B)
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
def train(data_loader, model, optimizer, epoch):
losses = AverageMeter()
accuracy = AverageMeter()
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.train()
global iteration
for idx, input_seq in enumerate(data_loader):
tic = time.time()
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[score_, mask_] = model(input_seq)
# visualize
if (iteration == 0) or (iteration == args.print_freq):
if B > 2: input_seq = input_seq[0:2,:]
writer_train.add_image('input_seq',
de_normalize(vutils.make_grid(
input_seq.transpose(2,3).contiguous().view(-1,3,args.img_dim,args.img_dim),
nrow=args.num_seq*args.seq_len)),
iteration)
del input_seq
if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_)
# score is a 6d tensor: [B, P, SQ, B2, N, SQ]
# similarity matrix is computed inside each gpu, thus here B == num_gpu * B2
score_flattened = score_.contiguous().view(B*NP*SQ, B2*NS*SQ)
target_flattened = target_.contiguous().view(B*NP*SQ, B2*NS*SQ)
target_flattened = target_flattened.double().argmax(dim=1)
loss = criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(score_flattened, target_flattened, (1,3,5))
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
losses.update(loss.item(), B)
accuracy.update(top1.item(), B)
del score_
optimizer.zero_grad()
loss.backward()
optimizer.step()
del loss
if idx % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t'
'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'.format(
epoch, idx, len(data_loader), top1, top3, top5, time.time()-tic, loss=losses))
writer_train.add_scalar('local/loss', losses.val, iteration)
writer_train.add_scalar('local/accuracy', accuracy.val, iteration)
iteration += 1
return losses.local_avg, accuracy.local_avg, [i.local_avg for i in accuracy_list]
def test(data_loader, model):
losses = AverageMeter()
acc_top1 = AverageMeter()
acc_top5 = AverageMeter()
confusion_mat = ConfusionMeter(args.num_class)
model.eval()
with torch.no_grad():
for idx, (input_seq, target) in tqdm(enumerate(data_loader),
total=len(data_loader)):
input_seq = input_seq.to(cuda)
target = target.to(cuda)
B = input_seq.size(0)
h0 = model.module.init_hidden(B)
input_seq = input_seq.squeeze(0) # squeeze the '1' batch dim
output, _ = model(input_seq, h0)
del input_seq
top1, top5 = calc_topk_accuracy(
torch.mean(torch.mean(nn.functional.softmax(output, 2), 0),
0,
keepdim=True), target, (1, 5))
acc_top1.update(top1.item(), B)
acc_top5.update(top5.item(), B)
del top1, top5
output = torch.mean(torch.mean(output, 0), 0, keepdim=True)
loss = criterion(output, target.squeeze(-1))
losses.update(loss.item(), B)
del loss
_, pred = torch.max(output, 1)
confusion_mat.update(pred, target.view(-1).byte())
print('Loss {loss.avg:.4f}\t'
'Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'.format(
loss=losses, top1=acc_top1, top5=acc_top5))
confusion_mat.plot_mat(args.test + '.svg')
write_log(
content=
'Loss {loss.avg:.4f}\t Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'
.format(loss=losses, top1=acc_top1, top5=acc_top5, args=args),
epoch=num_epoch,
filename=os.path.dirname(args.test) + '/test_log.md')
import ipdb
ipdb.set_trace()
return losses.avg, [acc_top1.avg, acc_top5.avg]
def test(data_loader, model, extensive=False):
losses = AverageMeter()
acc_top1 = AverageMeter()
acc_top5 = AverageMeter()
acc_table = AccuracyTable(data_loader.dataset.action_dict_decode)
confusion_mat = ConfusionMeter(args.num_class)
probs = {}
model.eval()
with torch.no_grad():
tq = tqdm(data_loader, desc="Test progress: ")
for idx, (input_seq, target, index) in enumerate(tq):
input_seq = input_seq.to(cuda)
target = target.to(cuda)
B = input_seq.size(0)
input_seq = input_seq.squeeze(0) # squeeze the '1' batch dim
output, _ = model(input_seq)
del input_seq
prob = torch.mean(torch.mean(nn.functional.softmax(output, 2), 0),
0,
keepdim=True)
top1, top5 = calc_topk_accuracy(prob, target, (1, 5))
acc_top1.update(top1.item(), B)
acc_top5.update(top5.item(), B)
del top1, top5
output = torch.mean(torch.mean(output, 0), 0, keepdim=True)
loss = criterion(output, target.squeeze(-1))
losses.update(loss.item(), B)
del loss
_, pred = torch.max(output, 1)
confusion_mat.update(pred, target.view(-1).byte())
acc_table.update(pred, target)
probs[index] = {
'prob': prob.detach().cpu(),
'target': target.detach().cpu()
}
tq.set_postfix({
'loss': losses.avg,
'acc1': acc_top1.avg,
'acc5': acc_top5.avg,
})
print('Test - Loss {loss.avg:.4f}\t'
'Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'.format(
loss=losses, top1=acc_top1, top5=acc_top5))
confusion_mat.plot_mat(args.test + '.svg')
write_log(
content=
'Loss {loss.avg:.4f}\t Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'
.format(loss=losses, top1=acc_top1, top5=acc_top5, args=args),
epoch=num_epoch,
filename=os.path.join(os.path.dirname(args.test),
'test_log_{}.md').format(args.notes))
with open(
os.path.join(os.path.dirname(args.test),
'test_probs_{}.pkl').format(args.notes), 'wb') as f:
pickle.dump(probs, f)
if extensive:
acc_table.print_table()
acc_table.print_dict()
# import ipdb; ipdb.set_trace()
return losses.avg, [acc_top1.avg, acc_top5.avg]
def validate(data_loader, model, epoch):
losses = AverageMeter()
accuracy = AverageMeter()
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.eval()
with torch.no_grad():
for idx, input_dict in tqdm(enumerate(data_loader),
total=len(data_loader)):
#print(idx, input_seq.shape)
input_seq = input_dict['t_seq']
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[final_score_, mask_, pred_radius_, score_] = model(input_seq)
del input_seq
m = nn.MSELoss()
loss = m(final_score_, torch.zeros_like(final_score_))
del final_score_
if args.distance == 'L2':
if idx == 0:
target_, (_, B2, NS, NP, SQ) = process_output(mask_)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.double()
_, target_flattened = target_flattened.max(dim=1)
top1, top3, top5 = calc_topk_accuracy(-score_,
target_flattened,
(1, 3, 5))
elif args.distance == 'cosine':
if idx == 0:
target_, (_, B2, NS, NP, SQ) = process_output(mask_)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.double()
_, target_flattened = target_flattened.max(dim=1)
top1, top3, top5 = calc_topk_accuracy(score_, target_flattened,
(1, 3, 5))
del score_, mask_
losses.update(loss.item(), B)
del loss
accuracy.update(top1.item(), B)
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
print('[{0}/{1}] Loss {loss.local_avg:.4f}\t'
'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f}; \t'
'radius {radius:.4f}; radius_var {radius_var:.4f}'.format(
epoch,
args.epochs,
*[i.avg for i in accuracy_list],
loss=losses,
radius=torch.mean(pred_radius_).detach().cpu().numpy(),
radius_var=torch.std(pred_radius_).detach().cpu().numpy()))
return [
losses.local_avg, accuracy.local_avg,
[i.local_avg for i in accuracy_list],
torch.mean(pred_radius_).detach().cpu().numpy(),
torch.std(pred_radius_).detach().cpu().numpy()
]
def train(data_loader, model, optimizer, epoch):
# average losses
losses = AverageMeter()
# average accuracy
accuracy = AverageMeter()
# top-1, top-3 and top-5 accuracy
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.train()
global iteration
#print (len(data_loader))
for idx, input_dict in enumerate(data_loader):
tic = time.time()
input_seq = input_dict['t_seq']
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[final_score_, mask_, pred_radius_, score_] = model(input_seq)
if args.loss_type == 'MSE':
m = nn.MSELoss()
loss = m(final_score_, torch.zeros_like(final_score_))
elif args.loss_type == 'L1':
loss = torch.sum(final_score_)
del final_score_
# visualize the input sequence to the network
if (iteration == 0) or (iteration == args.print_freq):
if B > 2:
input_seq = input_seq[0:2, :]
writer_train.add_image(
'input_seq',
de_normalize(
vutils.make_grid(input_seq.transpose(
2, 3).contiguous().view(-1, 3, args.img_dim,
args.img_dim),
nrow=args.num_seq * args.seq_len)),
iteration)
del input_seq
if args.distance == 'L2':
# why only for idx 0 ?
if idx == 0:
target_, (_, B2, NS, NP, SQ) = process_output(mask_)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
_, target_flattened = target_flattened.max(dim=1)
# loss function, here cross-entropy for DPC
top1, top3, top5 = calc_topk_accuracy(-score_, target_flattened,
(1, 3, 5))
elif args.distance == 'cosine':
# why only for idx 0 ?
if idx == 0:
target_, (_, B2, NS, NP, SQ) = process_output(mask_)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.double()
_, target_flattened = target_flattened.max(dim=1)
# loss function, here cross-entropy for DPC
top1, top3, top5 = calc_topk_accuracy(score_, target_flattened,
(1, 3, 5))
del mask_
# break
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
if args.loss_type == 'L1':
losses.update(loss.item() / len(score_)**2, B)
else:
losses.update(loss.item(), B)
accuracy.update(top1.item(), B)
del score_
optimizer.zero_grad()
loss.backward()
optimizer.step()
del loss
if idx % args.print_freq == 0:
print(
'Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t'
'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'
'radius: {radius:.4f}; radius_var: {radius_var:.4f}'.format(
epoch,
idx,
len(data_loader),
top1,
top3,
top5,
time.time() - tic,
loss=losses,
radius=torch.mean(pred_radius_).detach().cpu().numpy(),
radius_var=torch.std(pred_radius_).detach().cpu().numpy()))
writer_train.add_scalar('local/loss', losses.val, iteration)
writer_train.add_scalar('local/accuracy', accuracy.val, iteration)
# TODO plot radius etc also in tensorboard
iteration += 1
return [
losses.local_avg, accuracy.local_avg,
[i.local_avg for i in accuracy_list],
torch.mean(pred_radius_).detach().cpu().numpy(),
torch.std(pred_radius_).detach().cpu().numpy()
]
def train_step(self, args, idx, input_seq, iteration, epoch, writer_train,
losses, accuracy, accuracy_list):
self.model.train()
self.criterion.train()
self.optimizer.zero_grad()
tic = time.time()
input_seq = self._prepare_sample(input_seq)
B = input_seq.size(0)
[score_, mask_] = self.model(input_seq)
# visualize
if (iteration == 0) or (iteration == args.print_freq):
if B > 2: input_seq = input_seq[0:2, :]
writer_train.add_image(
'input_seq',
self.de_normalize(
vutils.make_grid(input_seq.transpose(
2, 3).contiguous().view(-1, 3, args.img_dim,
args.img_dim),
nrow=args.num_seq * args.seq_len)),
iteration)
del input_seq
if idx == 0:
self.target_, (_, self.B2, self.NS, self.NP,
self.SQ) = process_output(mask_)
score_flattened = score_.view(B * self.NP * self.SQ,
self.B2 * self.NS * self.SQ)
target_flattened = self.target_.view(B * self.NP * self.SQ,
self.B2 * self.NS * self.SQ)
target_flattened = target_flattened.long().argmax(dim=1)
loss = self.criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(score_flattened,
target_flattened, (1, 3, 5))
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
losses.update(loss.item(), B)
accuracy.update(top1.item(), B)
del score_
loss.backward()
self.optimizer.step()
del loss
if idx % args.print_freq == 0:
print('Epoch: [{0}][{1}]\t'
'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t'
'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f} T:{5:.2f}\t'.
format(epoch,
idx,
top1,
top3,
top5,
time.time() - tic,
loss=losses))
writer_train.add_scalar('local/loss', losses.val, iteration)
writer_train.add_scalar('local/accuracy', accuracy.val, iteration)
iteration += 1
def train(data_loader, model, optimizer, epoch):
# average losses
losses = AverageMeter()
# average accuracy
accuracy = AverageMeter()
# top-1, top-3 and top-5 accuracy
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.train()
global iteration
# for embeddings, get the label too
for idx, input_seq in enumerate(data_loader):
tic = time.time()
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[score_, mask_] = model(input_seq)
# visualize the input sequence to the network
if (iteration == 0) or (iteration == args.print_freq):
if B > 2:
input_seq = input_seq[0:2, :]
writer_train.add_image(
'input_seq',
de_normalize(
vutils.make_grid(input_seq.transpose(
2, 3).contiguous().view(-1, 3, args.img_dim,
args.img_dim),
nrow=args.num_seq * args.seq_len)),
iteration)
del input_seq
# why only for idx 0 ?
if idx == 0:
target_, (_, B2, NS, NP, SQ) = process_output(mask_)
# score is a 6d tensor: [B, P, SQ, B, N, SQ]
# Number Predicted NP, Number Sequence NS
score_flattened = score_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.double()
# print('score: ', score_flattened[0:4][0:4])
# print('target: ', target_flattened[0:4][0:4])
# print(score_flattened.shape)
# print(target_flattened.shape)
# added
if args.loss_function == 'CE':
target_flattened = target_flattened.double()
target_flattened = target_flattened.argmax(dim=1)
loss = criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(score_flattened,
target_flattened, (1, 3, 5))
elif args.loss_function == 'MSE':
target_flattened = target_flattened.float()
loss = criterion(score_flattened, target_flattened)
top1, top3, top5 = calc_topk_accuracy(
score_flattened, target_flattened.argmax(dim=1), (1, 3, 5))
# loss function, here cross-entropy for DPC
#print (score_flattened.shape, target_flattened)
accuracy_list[0].update(top1.item(), B)
accuracy_list[1].update(top3.item(), B)
accuracy_list[2].update(top5.item(), B)
losses.update(loss.item(), B)
accuracy.update(top1.item(), B)
del score_
optimizer.zero_grad()
loss.backward()
optimizer.step()
del loss
if idx % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t'
'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'.
format(epoch,
idx,
len(data_loader),
top1,
top3,
top5,
time.time() - tic,
loss=losses))
writer_train.add_scalar('local/loss', losses.val, iteration)
writer_train.add_scalar('local/accuracy', accuracy.val, iteration)
iteration += 1
# average loss, average accuracy and average accuracy for this
return losses.local_avg, accuracy.local_avg, [
i.local_avg for i in accuracy_list
]
def validate(data_loader, model, epoch):
losses_cpc = AverageMeter()
accuracy_cpc = AverageMeter()
accuracy_list_cpc = [AverageMeter(), AverageMeter(), AverageMeter()]
losses_hd = AverageMeter()
model.eval()
with torch.no_grad():
for idx, (input_seq, targets) in tqdm(enumerate(data_loader),
total=len(data_loader)):
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[score_, mask_], y_hd = model(input_seq)
del input_seq
if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_)
# [B, P, SQ, B, N, SQ]
score_flattened = score_.contiguous().view(B * NP * SQ,
B2 * NS * SQ)
target_flattened = target_.contiguous().view(
B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.double()
target_flattened = target_flattened.argmax(dim=1)
loss_cpc = criterion(score_flattened, target_flattened)
top1_cpc, top3_cpc, top5_cpc = calc_topk_accuracy(
score_flattened, target_flattened, (1, 3, 5))
losses_cpc.update(loss_cpc.item(), B)
accuracy_cpc.update(top1_cpc.item(), B)
accuracy_list_cpc[0].update(top1_cpc.item(), B)
accuracy_list_cpc[1].update(top3_cpc.item(), B)
accuracy_list_cpc[2].update(top5_cpc.item(), B)
if args.dataset == 'tdw':
if args.target == 'obj_categ':
y_gt = targets['category']
elif args.target == 'self_motion':
norm_cam = torch.linalg.norm(torch.cat(
(200 * targets['camera_motion']['translation']
['x_v'].unsqueeze(1), 200 * targets['camera_motion']
['translation']['z_v'].unsqueeze(1)),
dim=1),
dim=1)
norm_camobj = torch.linalg.norm(torch.cat(
(targets['camera_object_vec']['x'].unsqueeze(1),
targets['camera_object_vec']['z'].unsqueeze(1)),
dim=1),
dim=1)
y_gt = torch.cat((
(200 * targets['camera_motion']['translation']['x_v'] /
norm_cam - targets['camera_object_vec']['x'] /
norm_camobj).unsqueeze(dim=1),
(200 * targets['camera_motion']['translation']['z_v'] /
norm_cam - targets['camera_object_vec']['z'] /
norm_camobj).unsqueeze(dim=1),
targets['camera_motion']['rotation']['yaw'].unsqueeze(
dim=1)),
dim=1)
elif args.dataset == 'ucf101':
y_gt = targets - 1
else:
y_gt = targets
tic = time.time()
y_hd_gt = y_gt.squeeze().to(cuda)
loss_hd = criterion_aux(y_hd, y_hd_gt)
if isinstance(criterion_aux, nn.CrossEntropyLoss):
top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2))
accuracy_list[0].update(top1.item(), B)
accuracy.update(top1.item(), B)
elif isinstance(criterion_aux, nn.L1Loss) or isinstance(
criterion_aux, nn.MSELoss):
loss_hd = loss_hd / B
losses_hd.update(loss_hd.item(), B)
print('[{0}/{1}] CPC Loss {loss_cpc.local_avg:.4f}\t'
'Heading Loss {loss_hd.local_avg:.4f}\t'
'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f} \t'.format(
epoch,
args.epochs,
*[i.avg for i in accuracy_list_cpc],
loss_cpc=losses_cpc,
loss_hd=losses_hd))
return losses_cpc.local_avg, accuracy_cpc.local_avg, [
i.local_avg for i in accuracy_list_cpc
], losses_hd.local_avg
def train(data_loader, model, optimizer, epoch):
losses_cpc = AverageMeter()
accuracy_cpc = AverageMeter()
accuracy_list_cpc = [AverageMeter(), AverageMeter(), AverageMeter()]
losses_hd = AverageMeter()
model.train()
global iteration
for idx, (input_seq, targets) in enumerate(data_loader):
tic = time.time()
input_seq = input_seq.to(cuda)
B = input_seq.size(0)
[score_, mask_], y_hd = model(input_seq)
# visualize
if (iteration == 0) or (iteration == args.print_freq):
if B > 2: input_seq = input_seq[0:2, :]
writer_train.add_image(
'input_seq',
de_normalize(
vutils.make_grid(input_seq.transpose(
2, 3).contiguous().view(-1, 3, args.img_dim,
args.img_dim),
nrow=args.num_seq * args.seq_len)),
iteration)
del input_seq
if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_)
# score is a 6d tensor: [B, P, SQ, B, N, SQ]
score_flattened = score_.contiguous().view(B * NP * SQ, B2 * NS * SQ)
target_flattened = target_.contiguous().reshape(
B * NP * SQ, B2 * NS * SQ)
target_flattened = target_flattened.double()
target_flattened = target_flattened.argmax(dim=1)
score_flattened /= temperature
loss_cpc = criterion(score_flattened, target_flattened)
top1_cpc, top3_cpc, top5_cpc = calc_topk_accuracy(
score_flattened, target_flattened, (1, 3, 5))
accuracy_list_cpc[0].update(top1_cpc.item(), B)
accuracy_list_cpc[1].update(top3_cpc.item(), B)
accuracy_list_cpc[2].update(top5_cpc.item(), B)
losses_cpc.update(loss_cpc.item(), B)
accuracy_cpc.update(top1_cpc.item(), B)
del score_
optimizer.zero_grad()
loss_cpc.backward()
# optimizer.step()
del loss_cpc
if args.dataset == 'tdw':
if args.target == 'obj_categ':
y_gt = targets['category']
elif args.target == 'self_motion':
norm_cam = torch.linalg.norm(torch.cat(
(200 * targets['camera_motion']['translation']
['x_v'].unsqueeze(1), 200 * targets['camera_motion']
['translation']['z_v'].unsqueeze(1)),
dim=1),
dim=1)
norm_camobj = torch.linalg.norm(torch.cat(
(targets['camera_object_vec']['x'].unsqueeze(1),
targets['camera_object_vec']['z'].unsqueeze(1)),
dim=1),
dim=1)
y_gt = torch.cat(
((200 * targets['camera_motion']['translation']['x_v'] /
norm_cam - targets['camera_object_vec']['x'] /
norm_camobj).unsqueeze(dim=1),
(200 * targets['camera_motion']['translation']['z_v'] /
norm_cam - targets['camera_object_vec']['z'] /
norm_camobj).unsqueeze(dim=1),
targets['camera_motion']['rotation']['yaw'].unsqueeze(
dim=1)),
dim=1) #
elif args.dataset == 'ucf101':
y_gt = targets - 1
else:
y_gt = targets
y_gt = y_gt.squeeze().to(cuda)
loss_hd = criterion_aux(y_hd, y_gt)
if isinstance(criterion_aux, nn.CrossEntropyLoss):
top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2))
accuracy_list[0].update(top1.item(), B)
accuracy.update(top1.item(), B)
elif isinstance(criterion_aux, nn.L1Loss) or isinstance(
criterion_aux, nn.MSELoss):
loss_hd = loss_hd / B
losses_hd.update(loss_hd.item(), B)
# optimizer.zero_grad()
loss_hd_weighted = args.hd_weight * loss_hd
for name, param in model.module.backbone.named_parameters():
param.requires_grad = True #False
for name, param in model.module.agg.named_parameters():
param.requires_grad = True #False
for name, param in model.module.network_pred.named_parameters():
param.requires_grad = True #False
## **** this needs to be corrected - loss_hd is not weighted currently **** ####
loss_hd.backward()
optimizer.step()
del loss_hd
for name, param in model.module.backbone.named_parameters():
param.requires_grad = True
for name, param in model.module.agg.named_parameters():
param.requires_grad = True
for name, param in model.module.network_pred.named_parameters():
param.requires_grad = True
if idx % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'CPC Loss {loss_cpc.val:.6f} ({loss_cpc.local_avg:.4f})\t'
'Heading Loss {loss_hd.val:.6f} ({loss_hd.local_avg:.4f})\t'
'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'.
format(epoch,
idx,
len(data_loader),
top1_cpc,
top3_cpc,
top5_cpc,
time.time() - tic,
loss_cpc=losses_cpc,
loss_hd=losses_hd))
writer_train.add_scalar('local/loss', losses_cpc.val, iteration)
writer_train.add_scalar('local/accuracy', accuracy_cpc.val,
iteration)
iteration += 1
return losses_cpc.local_avg, accuracy_cpc.local_avg, [
i.local_avg for i in accuracy_list_cpc
], losses_hd.local_avg
def update_metrics(self, gt_all_features, flat_scores, stats, data):
B, NS, N = self.args["batch_size"], self.args["pred_step"], self.args["num_seq"]
loss_dict = {k: {} for k in self.losses}
contexts = {}
for mode in self.modes:
SQ = self.models[mode].last_size ** 2
target_flattened = self.standard_grid_mask[mode].view(self.B0 * NS * SQ, self.B1 * NS * SQ)
# CPC loss
if True:
score_flat = flat_scores[mode]
target = target_flattened
target_lbl = target.float().argmax(dim=1)
# Compute and log performance metrics
topKs = calc_topk_accuracy(score_flat, target_lbl, self.accuracyKList)
for i in range(len(self.accuracyKList)):
stats[mu.CPCLoss][mode]["acc" + str(self.accuracyKList[i])].update(topKs[i].item(), B)
# Compute CPC loss for independent model training
loss_dict[mu.CPCLoss][mode] = self.criterias[mu.CPCLoss](score_flat, target)
for (m0, m1) in self.mode_pairs:
tupName = self.get_tuple_name(m0, m1)
# Cdot related losses
if True:
comp_gt_all0 = self.compiled_features[m0](gt_all_features[m0]).unsqueeze(3).unsqueeze(3)
comp_gt_all1 = self.compiled_features[m1](gt_all_features[m1]).unsqueeze(3).unsqueeze(3)
cdot0 = self.interModeDotHandler.get_cluster_dots(comp_gt_all0)
cdot1 = self.interModeDotHandler.get_cluster_dots(comp_gt_all1)
B, NS, B2, NS = cdot0.shape
assert cdot0.shape == cdot1.shape == (B, NS, B2, NS), \
"Invalid shapes: {}, {}, {}".format(cdot0.shape, cdot1.shape, (B, NS, B2, NS))
cos_sim_dot_loss = self.criterias[self.CooperativeLossLabel](cdot0, cdot1)
loss_dict[self.CooperativeLossLabel][tupName] = self.dot_wt * cos_sim_dot_loss
# Modality sync loss
if True:
sync_loss, mode_stats = self.modeSyncers[tupName](gt_all_features[m0], gt_all_features[m1])
# stats: dict modeLoss -> specificStat
for modeLoss in mode_stats.keys():
for stat in mode_stats[modeLoss].keys():
stats[modeLoss][tupName][stat].update(mode_stats[modeLoss][stat].item(), B)
loss_dict[mu.ModeSim][tupName] = self.sync_wt * sync_loss
return loss_dict, stats
def validate(data_loader, model, epoch, criterion):
losses = AverageMeter()
accuracy = AverageMeter()
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.eval()
with torch.no_grad():
for idx, (input_seq, targets) in enumerate(data_loader):
if args.dataset == 'tdw':
if args.target == 'obj_categ':
y_gt = targets['category']
elif args.target == 'self_motion':
# m_tmp, theta_tmp = cart2pol(targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x'],
# targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z'])
# y_gt = torch.cat((m_tmp.unsqueeze(dim = 1),
# theta_tmp.unsqueeze(dim = 1),
# targets['camera_motion']['rotation']['yaw'].unsqueeze(dim = 1)),
# dim = 1) #
norm_cam = torch.linalg.norm(torch.cat(
(100 * targets['camera_motion']['translation']
['x_v'].unsqueeze(1), 100 * targets['camera_motion']
['translation']['z_v'].unsqueeze(1)),
dim=1),
dim=1)
norm_camobj = torch.linalg.norm(torch.cat(
(targets['camera_object_vec']['x'].unsqueeze(1),
targets['camera_object_vec']['z'].unsqueeze(1)),
dim=1),
dim=1)
y_gt = torch.cat((
(100 * targets['camera_motion']['translation']['x_v'] /
norm_cam - targets['camera_object_vec']['x'] /
norm_camobj).unsqueeze(dim=1),
(100 * targets['camera_motion']['translation']['z_v'] /
norm_cam - targets['camera_object_vec']['z'] /
norm_camobj).unsqueeze(dim=1),
targets['camera_motion']['rotation']['yaw'].unsqueeze(
dim=1)),
dim=1) #
# y_gt = torch.cat(((100*targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x']).unsqueeze(dim = 1),
# (100*targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z']).unsqueeze(dim = 1)
# ),
# dim = 1) #
# heading_speed, _ = cart2pol(targets['camera_motion']['translation']['x_v'], targets['camera_motion']['translation']['z_v'])
# _, theta_1 = cart2pol(targets['camera_object_vec']['x'], targets['camera_object_vec']['z'])
# _, theta_2 = cart2pol(targets['camera_motion']['translation']['x_v'],targets['camera_motion']['translation']['z_v'])
# heading_theta = theta_1 - theta_2
# head_rotation_yaw = targets['camera_motion']['rotation']['yaw']
# y_gt = torch.cat((
# head_rotation_yaw.unsqueeze(dim = 1),
# 100*heading_speed.unsqueeze(dim = 1) - 1.5),
# dim = 1) #
elif args.dataset == 'ucf101':
y_gt = targets - 1
else:
y_gt = targets
tic = time.time()
y_gt = y_gt.squeeze().to(cuda)
input_seq = input_seq.squeeze().to(cuda)
B = input_seq.size(0)
y = model(input_seq)
del input_seq
loss = criterion(y, y_gt)
if isinstance(criterion, nn.CrossEntropyLoss):
top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2))
accuracy_list[0].update(top1.item(), B)
accuracy.update(top1.item(), B)
elif isinstance(criterion, nn.L1Loss) or isinstance(
criterion, nn.MSELoss):
loss = loss / B
losses.update(loss.item(), B)
if isinstance(criterion, nn.CrossEntropyLoss):
print('[{0}/{1}] Loss {loss.local_avg:.4f}\t'
'Acc: top1 {2:.4f} \t'.format(epoch,
args.epochs,
*[i.avg for i in accuracy_list],
loss=losses))
elif isinstance(criterion, nn.L1Loss) or isinstance(criterion, nn.MSELoss):
print('[{0}/{1}] Loss {loss.local_avg:.4f}\t'.format(epoch,
args.epochs,
loss=losses))
return losses.local_avg, accuracy.local_avg, [
i.local_avg for i in accuracy_list
]
def train(data_loader, model, optimizer, epoch, criterion):
losses = AverageMeter()
accuracy = AverageMeter()
accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()]
model.train()
global iteration
for idx, (input_seq, targets) in enumerate(data_loader):
# print(idx)
if args.dataset == 'tdw':
if args.target == 'obj_categ':
y_gt = targets['category']
elif args.target == 'self_motion':
# m_tmp, theta_tmp = cart2pol(targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x'],
# targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z'])
# y_gt = torch.cat((m_tmp.unsqueeze(dim = 1),
# theta_tmp.unsqueeze(dim = 1),
# targets['camera_motion']['rotation']['yaw'].unsqueeze(dim = 1)),
# dim = 1) #
norm_cam = torch.linalg.norm(torch.cat(
(100 * targets['camera_motion']['translation']
['x_v'].unsqueeze(1), 100 * targets['camera_motion']
['translation']['z_v'].unsqueeze(1)),
dim=1),
dim=1)
norm_camobj = torch.linalg.norm(torch.cat(
(targets['camera_object_vec']['x'].unsqueeze(1),
targets['camera_object_vec']['z'].unsqueeze(1)),
dim=1),
dim=1)
y_gt = torch.cat(
((100 * targets['camera_motion']['translation']['x_v'] /
norm_cam - targets['camera_object_vec']['x'] /
norm_camobj).unsqueeze(dim=1),
(100 * targets['camera_motion']['translation']['z_v'] /
norm_cam - targets['camera_object_vec']['z'] /
norm_camobj).unsqueeze(dim=1),
targets['camera_motion']['rotation']['yaw'].unsqueeze(
dim=1)),
dim=1) #
# y_gt = torch.cat(((100*targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x']).unsqueeze(dim = 1),
# (100*targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z']).unsqueeze(dim = 1)
# ),
# dim = 1) #
# heading_speed, _ = cart2pol(targets['camera_motion']['translation']['x_v'], targets['camera_motion']['translation']['z_v'])
# _, theta_1 = cart2pol(targets['camera_object_vec']['x'], targets['camera_object_vec']['z'])
# _, theta_2 = cart2pol(targets['camera_motion']['translation']['x_v'],targets['camera_motion']['translation']['z_v'])
# heading_theta = theta_2 - theta_1
# head_rotation_yaw = targets['camera_motion']['rotation']['yaw']
# y_gt = torch.cat((
# head_rotation_yaw.unsqueeze(dim = 1),
# 100*heading_speed.unsqueeze(dim = 1) - 1.5),
# dim = 1) #
elif args.dataset == 'ucf101':
y_gt = targets - 1
else:
y_gt = targets
tic = time.time()
y_gt = y_gt.squeeze().to(cuda)
input_seq = input_seq.squeeze().to(cuda)
B = input_seq.size(0)
y = model(input_seq)
del input_seq
loss = criterion(y, y_gt)
if isinstance(criterion, nn.CrossEntropyLoss):
top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2))
accuracy_list[0].update(top1.item(), B)
accuracy.update(top1.item(), B)
elif isinstance(criterion, nn.L1Loss) or isinstance(
criterion, nn.MSELoss):
loss = loss / B
losses.update(loss.item(), B)
optimizer.zero_grad()
loss.backward()
optimizer.step()
del loss
if idx % args.print_freq == 0:
if isinstance(criterion, nn.CrossEntropyLoss):
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t'
'Acc: top1 {3:.4f}; T:{4:.2f}\t'.format(epoch,
idx,
len(data_loader),
top1,
time.time() -
tic,
loss=losses))
elif isinstance(criterion, nn.L1Loss) or isinstance(
criterion, nn.MSELoss):
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t'
'T:{3:.2f}\t'.format(epoch,
idx,
len(data_loader),
time.time() - tic,
loss=losses))
iteration += 1
return losses.local_avg, accuracy.local_avg, [
i.local_avg for i in accuracy_list
]
