def validate(data_loader, model, epoch): losses = AverageMeter() accuracy = AverageMeter() accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.eval() with torch.no_grad(): for idx, input_seq in tqdm(enumerate(data_loader), total=len(data_loader)): input_seq = input_seq.to(cuda) B = input_seq.size(0) [score_, mask_] = model(input_seq) del input_seq if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) # [B, P, SQ, B, N, SQ] score_flattened = score_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) if args.loss_function == 'CE': target_flattened = target_flattened.double() target_flattened = target_flattened.argmax(dim=1) loss = criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy(score_flattened, target_flattened, (1, 3, 5)) elif args.loss_function == 'MSE': target_flattened = target_flattened.float() loss = criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy( score_flattened, target_flattened.argmax(dim=1), (1, 3, 5)) losses.update(loss.item(), B) accuracy.update(top1.item(), B) accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B) print('[{0}/{1}] Loss {loss.local_avg:.4f}\t' 'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f} \t'.format( epoch, args.epochs, *[i.avg for i in accuracy_list], loss=losses)) return losses.local_avg, accuracy.local_avg, [ i.local_avg for i in accuracy_list ]
def ensemble(prob_imgs=None, prob_flow=None, prob_seg=None, prob_kphm=None): acc_top1 = AverageMeter() acc_top5 = AverageMeter() probs = [prob_imgs, prob_flow, prob_seg, prob_kphm] for idx in range(len(probs)): if probs[idx] is not None: probs[idx] = {k[0][0].data: v for k, v in probs[idx].items()} valid_probs = [x for x in probs if x is not None] weights = [2, 2, 1, 1] ovr_probs = {} for k in valid_probs[0].keys(): ovr_probs[k] = valid_probs[0][k]['prob'] * 0.0 total = 0 for idx in range(len(probs)): p = probs[idx] if p is not None: total += weights[idx] ovr_probs[k] += p[k]['prob'] * weights[idx] ovr_probs[k] /= total top1, top5 = calc_topk_accuracy(ovr_probs[k], valid_probs[0][k]['target'], (1, 5)) acc_top1.update(top1.item(), 1) acc_top5.update(top5.item(), 1) print('Test - Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'.format( top1=acc_top1, top5=acc_top5))
def valid_step(self, idx, input_seq, losses, accuracy, accuracy_list): self.model.eval() self.criterion.eval() input_seq = self._prepare_sample(input_seq) B = input_seq.size(0) [score_, mask_] = self.model(input_seq) del input_seq if idx == 0: target_, (_, self.B2, NS, NP, SQ) = process_output(mask_) # [B, P, SQ, B, N, SQ] score_flattened = score_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.argmax(dim=1) loss = self.criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy(score_flattened, target_flattened, (1, 3, 5)) losses.update(loss.item(), B) accuracy.update(top1.item(), B) accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B)
def train(data_loader, model, optimizer, epoch): losses = AverageMeter() accuracy = AverageMeter() accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.train() global iteration for idx, input_seq in enumerate(data_loader): tic = time.time() input_seq = input_seq.to(cuda) B = input_seq.size(0) [score_, mask_] = model(input_seq) # visualize if (iteration == 0) or (iteration == args.print_freq): if B > 2: input_seq = input_seq[0:2,:] writer_train.add_image('input_seq', de_normalize(vutils.make_grid( input_seq.transpose(2,3).contiguous().view(-1,3,args.img_dim,args.img_dim), nrow=args.num_seq*args.seq_len)), iteration) del input_seq if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) # score is a 6d tensor: [B, P, SQ, B2, N, SQ] # similarity matrix is computed inside each gpu, thus here B == num_gpu * B2 score_flattened = score_.contiguous().view(B*NP*SQ, B2*NS*SQ) target_flattened = target_.contiguous().view(B*NP*SQ, B2*NS*SQ) target_flattened = target_flattened.double().argmax(dim=1) loss = criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy(score_flattened, target_flattened, (1,3,5)) accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B) losses.update(loss.item(), B) accuracy.update(top1.item(), B) del score_ optimizer.zero_grad() loss.backward() optimizer.step() del loss if idx % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t' 'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'.format( epoch, idx, len(data_loader), top1, top3, top5, time.time()-tic, loss=losses)) writer_train.add_scalar('local/loss', losses.val, iteration) writer_train.add_scalar('local/accuracy', accuracy.val, iteration) iteration += 1 return losses.local_avg, accuracy.local_avg, [i.local_avg for i in accuracy_list]
def test(data_loader, model): losses = AverageMeter() acc_top1 = AverageMeter() acc_top5 = AverageMeter() confusion_mat = ConfusionMeter(args.num_class) model.eval() with torch.no_grad(): for idx, (input_seq, target) in tqdm(enumerate(data_loader), total=len(data_loader)): input_seq = input_seq.to(cuda) target = target.to(cuda) B = input_seq.size(0) h0 = model.module.init_hidden(B) input_seq = input_seq.squeeze(0) # squeeze the '1' batch dim output, _ = model(input_seq, h0) del input_seq top1, top5 = calc_topk_accuracy( torch.mean(torch.mean(nn.functional.softmax(output, 2), 0), 0, keepdim=True), target, (1, 5)) acc_top1.update(top1.item(), B) acc_top5.update(top5.item(), B) del top1, top5 output = torch.mean(torch.mean(output, 0), 0, keepdim=True) loss = criterion(output, target.squeeze(-1)) losses.update(loss.item(), B) del loss _, pred = torch.max(output, 1) confusion_mat.update(pred, target.view(-1).byte()) print('Loss {loss.avg:.4f}\t' 'Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'.format( loss=losses, top1=acc_top1, top5=acc_top5)) confusion_mat.plot_mat(args.test + '.svg') write_log( content= 'Loss {loss.avg:.4f}\t Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t' .format(loss=losses, top1=acc_top1, top5=acc_top5, args=args), epoch=num_epoch, filename=os.path.dirname(args.test) + '/test_log.md') import ipdb ipdb.set_trace() return losses.avg, [acc_top1.avg, acc_top5.avg]
def test(data_loader, model, extensive=False): losses = AverageMeter() acc_top1 = AverageMeter() acc_top5 = AverageMeter() acc_table = AccuracyTable(data_loader.dataset.action_dict_decode) confusion_mat = ConfusionMeter(args.num_class) probs = {} model.eval() with torch.no_grad(): tq = tqdm(data_loader, desc="Test progress: ") for idx, (input_seq, target, index) in enumerate(tq): input_seq = input_seq.to(cuda) target = target.to(cuda) B = input_seq.size(0) input_seq = input_seq.squeeze(0) # squeeze the '1' batch dim output, _ = model(input_seq) del input_seq prob = torch.mean(torch.mean(nn.functional.softmax(output, 2), 0), 0, keepdim=True) top1, top5 = calc_topk_accuracy(prob, target, (1, 5)) acc_top1.update(top1.item(), B) acc_top5.update(top5.item(), B) del top1, top5 output = torch.mean(torch.mean(output, 0), 0, keepdim=True) loss = criterion(output, target.squeeze(-1)) losses.update(loss.item(), B) del loss _, pred = torch.max(output, 1) confusion_mat.update(pred, target.view(-1).byte()) acc_table.update(pred, target) probs[index] = { 'prob': prob.detach().cpu(), 'target': target.detach().cpu() } tq.set_postfix({ 'loss': losses.avg, 'acc1': acc_top1.avg, 'acc5': acc_top5.avg, }) print('Test - Loss {loss.avg:.4f}\t' 'Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t'.format( loss=losses, top1=acc_top1, top5=acc_top5)) confusion_mat.plot_mat(args.test + '.svg') write_log( content= 'Loss {loss.avg:.4f}\t Acc top1: {top1.avg:.4f} Acc top5: {top5.avg:.4f} \t' .format(loss=losses, top1=acc_top1, top5=acc_top5, args=args), epoch=num_epoch, filename=os.path.join(os.path.dirname(args.test), 'test_log_{}.md').format(args.notes)) with open( os.path.join(os.path.dirname(args.test), 'test_probs_{}.pkl').format(args.notes), 'wb') as f: pickle.dump(probs, f) if extensive: acc_table.print_table() acc_table.print_dict() # import ipdb; ipdb.set_trace() return losses.avg, [acc_top1.avg, acc_top5.avg]
def validate(data_loader, model, epoch): losses = AverageMeter() accuracy = AverageMeter() accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.eval() with torch.no_grad(): for idx, input_dict in tqdm(enumerate(data_loader), total=len(data_loader)): #print(idx, input_seq.shape) input_seq = input_dict['t_seq'] input_seq = input_seq.to(cuda) B = input_seq.size(0) [final_score_, mask_, pred_radius_, score_] = model(input_seq) del input_seq m = nn.MSELoss() loss = m(final_score_, torch.zeros_like(final_score_)) del final_score_ if args.distance == 'L2': if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.double() _, target_flattened = target_flattened.max(dim=1) top1, top3, top5 = calc_topk_accuracy(-score_, target_flattened, (1, 3, 5)) elif args.distance == 'cosine': if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.double() _, target_flattened = target_flattened.max(dim=1) top1, top3, top5 = calc_topk_accuracy(score_, target_flattened, (1, 3, 5)) del score_, mask_ losses.update(loss.item(), B) del loss accuracy.update(top1.item(), B) accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B) print('[{0}/{1}] Loss {loss.local_avg:.4f}\t' 'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f}; \t' 'radius {radius:.4f}; radius_var {radius_var:.4f}'.format( epoch, args.epochs, *[i.avg for i in accuracy_list], loss=losses, radius=torch.mean(pred_radius_).detach().cpu().numpy(), radius_var=torch.std(pred_radius_).detach().cpu().numpy())) return [ losses.local_avg, accuracy.local_avg, [i.local_avg for i in accuracy_list], torch.mean(pred_radius_).detach().cpu().numpy(), torch.std(pred_radius_).detach().cpu().numpy() ]
def train(data_loader, model, optimizer, epoch): # average losses losses = AverageMeter() # average accuracy accuracy = AverageMeter() # top-1, top-3 and top-5 accuracy accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.train() global iteration #print (len(data_loader)) for idx, input_dict in enumerate(data_loader): tic = time.time() input_seq = input_dict['t_seq'] input_seq = input_seq.to(cuda) B = input_seq.size(0) [final_score_, mask_, pred_radius_, score_] = model(input_seq) if args.loss_type == 'MSE': m = nn.MSELoss() loss = m(final_score_, torch.zeros_like(final_score_)) elif args.loss_type == 'L1': loss = torch.sum(final_score_) del final_score_ # visualize the input sequence to the network if (iteration == 0) or (iteration == args.print_freq): if B > 2: input_seq = input_seq[0:2, :] writer_train.add_image( 'input_seq', de_normalize( vutils.make_grid(input_seq.transpose( 2, 3).contiguous().view(-1, 3, args.img_dim, args.img_dim), nrow=args.num_seq * args.seq_len)), iteration) del input_seq if args.distance == 'L2': # why only for idx 0 ? if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) _, target_flattened = target_flattened.max(dim=1) # loss function, here cross-entropy for DPC top1, top3, top5 = calc_topk_accuracy(-score_, target_flattened, (1, 3, 5)) elif args.distance == 'cosine': # why only for idx 0 ? if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.double() _, target_flattened = target_flattened.max(dim=1) # loss function, here cross-entropy for DPC top1, top3, top5 = calc_topk_accuracy(score_, target_flattened, (1, 3, 5)) del mask_ # break accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B) if args.loss_type == 'L1': losses.update(loss.item() / len(score_)**2, B) else: losses.update(loss.item(), B) accuracy.update(top1.item(), B) del score_ optimizer.zero_grad() loss.backward() optimizer.step() del loss if idx % args.print_freq == 0: print( 'Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t' 'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t' 'radius: {radius:.4f}; radius_var: {radius_var:.4f}'.format( epoch, idx, len(data_loader), top1, top3, top5, time.time() - tic, loss=losses, radius=torch.mean(pred_radius_).detach().cpu().numpy(), radius_var=torch.std(pred_radius_).detach().cpu().numpy())) writer_train.add_scalar('local/loss', losses.val, iteration) writer_train.add_scalar('local/accuracy', accuracy.val, iteration) # TODO plot radius etc also in tensorboard iteration += 1 return [ losses.local_avg, accuracy.local_avg, [i.local_avg for i in accuracy_list], torch.mean(pred_radius_).detach().cpu().numpy(), torch.std(pred_radius_).detach().cpu().numpy() ]
def train_step(self, args, idx, input_seq, iteration, epoch, writer_train, losses, accuracy, accuracy_list): self.model.train() self.criterion.train() self.optimizer.zero_grad() tic = time.time() input_seq = self._prepare_sample(input_seq) B = input_seq.size(0) [score_, mask_] = self.model(input_seq) # visualize if (iteration == 0) or (iteration == args.print_freq): if B > 2: input_seq = input_seq[0:2, :] writer_train.add_image( 'input_seq', self.de_normalize( vutils.make_grid(input_seq.transpose( 2, 3).contiguous().view(-1, 3, args.img_dim, args.img_dim), nrow=args.num_seq * args.seq_len)), iteration) del input_seq if idx == 0: self.target_, (_, self.B2, self.NS, self.NP, self.SQ) = process_output(mask_) score_flattened = score_.view(B * self.NP * self.SQ, self.B2 * self.NS * self.SQ) target_flattened = self.target_.view(B * self.NP * self.SQ, self.B2 * self.NS * self.SQ) target_flattened = target_flattened.long().argmax(dim=1) loss = self.criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy(score_flattened, target_flattened, (1, 3, 5)) accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B) losses.update(loss.item(), B) accuracy.update(top1.item(), B) del score_ loss.backward() self.optimizer.step() del loss if idx % args.print_freq == 0: print('Epoch: [{0}][{1}]\t' 'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t' 'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f} T:{5:.2f}\t'. format(epoch, idx, top1, top3, top5, time.time() - tic, loss=losses)) writer_train.add_scalar('local/loss', losses.val, iteration) writer_train.add_scalar('local/accuracy', accuracy.val, iteration) iteration += 1
def train(data_loader, model, optimizer, epoch): # average losses losses = AverageMeter() # average accuracy accuracy = AverageMeter() # top-1, top-3 and top-5 accuracy accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.train() global iteration # for embeddings, get the label too for idx, input_seq in enumerate(data_loader): tic = time.time() input_seq = input_seq.to(cuda) B = input_seq.size(0) [score_, mask_] = model(input_seq) # visualize the input sequence to the network if (iteration == 0) or (iteration == args.print_freq): if B > 2: input_seq = input_seq[0:2, :] writer_train.add_image( 'input_seq', de_normalize( vutils.make_grid(input_seq.transpose( 2, 3).contiguous().view(-1, 3, args.img_dim, args.img_dim), nrow=args.num_seq * args.seq_len)), iteration) del input_seq # why only for idx 0 ? if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) # score is a 6d tensor: [B, P, SQ, B, N, SQ] # Number Predicted NP, Number Sequence NS score_flattened = score_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_.view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.double() # print('score: ', score_flattened[0:4][0:4]) # print('target: ', target_flattened[0:4][0:4]) # print(score_flattened.shape) # print(target_flattened.shape) # added if args.loss_function == 'CE': target_flattened = target_flattened.double() target_flattened = target_flattened.argmax(dim=1) loss = criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy(score_flattened, target_flattened, (1, 3, 5)) elif args.loss_function == 'MSE': target_flattened = target_flattened.float() loss = criterion(score_flattened, target_flattened) top1, top3, top5 = calc_topk_accuracy( score_flattened, target_flattened.argmax(dim=1), (1, 3, 5)) # loss function, here cross-entropy for DPC #print (score_flattened.shape, target_flattened) accuracy_list[0].update(top1.item(), B) accuracy_list[1].update(top3.item(), B) accuracy_list[2].update(top5.item(), B) losses.update(loss.item(), B) accuracy.update(top1.item(), B) del score_ optimizer.zero_grad() loss.backward() optimizer.step() del loss if idx % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t' 'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'. format(epoch, idx, len(data_loader), top1, top3, top5, time.time() - tic, loss=losses)) writer_train.add_scalar('local/loss', losses.val, iteration) writer_train.add_scalar('local/accuracy', accuracy.val, iteration) iteration += 1 # average loss, average accuracy and average accuracy for this return losses.local_avg, accuracy.local_avg, [ i.local_avg for i in accuracy_list ]
def validate(data_loader, model, epoch): losses_cpc = AverageMeter() accuracy_cpc = AverageMeter() accuracy_list_cpc = [AverageMeter(), AverageMeter(), AverageMeter()] losses_hd = AverageMeter() model.eval() with torch.no_grad(): for idx, (input_seq, targets) in tqdm(enumerate(data_loader), total=len(data_loader)): input_seq = input_seq.to(cuda) B = input_seq.size(0) [score_, mask_], y_hd = model(input_seq) del input_seq if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) # [B, P, SQ, B, N, SQ] score_flattened = score_.contiguous().view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_.contiguous().view( B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.double() target_flattened = target_flattened.argmax(dim=1) loss_cpc = criterion(score_flattened, target_flattened) top1_cpc, top3_cpc, top5_cpc = calc_topk_accuracy( score_flattened, target_flattened, (1, 3, 5)) losses_cpc.update(loss_cpc.item(), B) accuracy_cpc.update(top1_cpc.item(), B) accuracy_list_cpc[0].update(top1_cpc.item(), B) accuracy_list_cpc[1].update(top3_cpc.item(), B) accuracy_list_cpc[2].update(top5_cpc.item(), B) if args.dataset == 'tdw': if args.target == 'obj_categ': y_gt = targets['category'] elif args.target == 'self_motion': norm_cam = torch.linalg.norm(torch.cat( (200 * targets['camera_motion']['translation'] ['x_v'].unsqueeze(1), 200 * targets['camera_motion'] ['translation']['z_v'].unsqueeze(1)), dim=1), dim=1) norm_camobj = torch.linalg.norm(torch.cat( (targets['camera_object_vec']['x'].unsqueeze(1), targets['camera_object_vec']['z'].unsqueeze(1)), dim=1), dim=1) y_gt = torch.cat(( (200 * targets['camera_motion']['translation']['x_v'] / norm_cam - targets['camera_object_vec']['x'] / norm_camobj).unsqueeze(dim=1), (200 * targets['camera_motion']['translation']['z_v'] / norm_cam - targets['camera_object_vec']['z'] / norm_camobj).unsqueeze(dim=1), targets['camera_motion']['rotation']['yaw'].unsqueeze( dim=1)), dim=1) elif args.dataset == 'ucf101': y_gt = targets - 1 else: y_gt = targets tic = time.time() y_hd_gt = y_gt.squeeze().to(cuda) loss_hd = criterion_aux(y_hd, y_hd_gt) if isinstance(criterion_aux, nn.CrossEntropyLoss): top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2)) accuracy_list[0].update(top1.item(), B) accuracy.update(top1.item(), B) elif isinstance(criterion_aux, nn.L1Loss) or isinstance( criterion_aux, nn.MSELoss): loss_hd = loss_hd / B losses_hd.update(loss_hd.item(), B) print('[{0}/{1}] CPC Loss {loss_cpc.local_avg:.4f}\t' 'Heading Loss {loss_hd.local_avg:.4f}\t' 'Acc: top1 {2:.4f}; top3 {3:.4f}; top5 {4:.4f} \t'.format( epoch, args.epochs, *[i.avg for i in accuracy_list_cpc], loss_cpc=losses_cpc, loss_hd=losses_hd)) return losses_cpc.local_avg, accuracy_cpc.local_avg, [ i.local_avg for i in accuracy_list_cpc ], losses_hd.local_avg
def train(data_loader, model, optimizer, epoch): losses_cpc = AverageMeter() accuracy_cpc = AverageMeter() accuracy_list_cpc = [AverageMeter(), AverageMeter(), AverageMeter()] losses_hd = AverageMeter() model.train() global iteration for idx, (input_seq, targets) in enumerate(data_loader): tic = time.time() input_seq = input_seq.to(cuda) B = input_seq.size(0) [score_, mask_], y_hd = model(input_seq) # visualize if (iteration == 0) or (iteration == args.print_freq): if B > 2: input_seq = input_seq[0:2, :] writer_train.add_image( 'input_seq', de_normalize( vutils.make_grid(input_seq.transpose( 2, 3).contiguous().view(-1, 3, args.img_dim, args.img_dim), nrow=args.num_seq * args.seq_len)), iteration) del input_seq if idx == 0: target_, (_, B2, NS, NP, SQ) = process_output(mask_) # score is a 6d tensor: [B, P, SQ, B, N, SQ] score_flattened = score_.contiguous().view(B * NP * SQ, B2 * NS * SQ) target_flattened = target_.contiguous().reshape( B * NP * SQ, B2 * NS * SQ) target_flattened = target_flattened.double() target_flattened = target_flattened.argmax(dim=1) score_flattened /= temperature loss_cpc = criterion(score_flattened, target_flattened) top1_cpc, top3_cpc, top5_cpc = calc_topk_accuracy( score_flattened, target_flattened, (1, 3, 5)) accuracy_list_cpc[0].update(top1_cpc.item(), B) accuracy_list_cpc[1].update(top3_cpc.item(), B) accuracy_list_cpc[2].update(top5_cpc.item(), B) losses_cpc.update(loss_cpc.item(), B) accuracy_cpc.update(top1_cpc.item(), B) del score_ optimizer.zero_grad() loss_cpc.backward() # optimizer.step() del loss_cpc if args.dataset == 'tdw': if args.target == 'obj_categ': y_gt = targets['category'] elif args.target == 'self_motion': norm_cam = torch.linalg.norm(torch.cat( (200 * targets['camera_motion']['translation'] ['x_v'].unsqueeze(1), 200 * targets['camera_motion'] ['translation']['z_v'].unsqueeze(1)), dim=1), dim=1) norm_camobj = torch.linalg.norm(torch.cat( (targets['camera_object_vec']['x'].unsqueeze(1), targets['camera_object_vec']['z'].unsqueeze(1)), dim=1), dim=1) y_gt = torch.cat( ((200 * targets['camera_motion']['translation']['x_v'] / norm_cam - targets['camera_object_vec']['x'] / norm_camobj).unsqueeze(dim=1), (200 * targets['camera_motion']['translation']['z_v'] / norm_cam - targets['camera_object_vec']['z'] / norm_camobj).unsqueeze(dim=1), targets['camera_motion']['rotation']['yaw'].unsqueeze( dim=1)), dim=1) # elif args.dataset == 'ucf101': y_gt = targets - 1 else: y_gt = targets y_gt = y_gt.squeeze().to(cuda) loss_hd = criterion_aux(y_hd, y_gt) if isinstance(criterion_aux, nn.CrossEntropyLoss): top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2)) accuracy_list[0].update(top1.item(), B) accuracy.update(top1.item(), B) elif isinstance(criterion_aux, nn.L1Loss) or isinstance( criterion_aux, nn.MSELoss): loss_hd = loss_hd / B losses_hd.update(loss_hd.item(), B) # optimizer.zero_grad() loss_hd_weighted = args.hd_weight * loss_hd for name, param in model.module.backbone.named_parameters(): param.requires_grad = True #False for name, param in model.module.agg.named_parameters(): param.requires_grad = True #False for name, param in model.module.network_pred.named_parameters(): param.requires_grad = True #False ## **** this needs to be corrected - loss_hd is not weighted currently **** #### loss_hd.backward() optimizer.step() del loss_hd for name, param in model.module.backbone.named_parameters(): param.requires_grad = True for name, param in model.module.agg.named_parameters(): param.requires_grad = True for name, param in model.module.network_pred.named_parameters(): param.requires_grad = True if idx % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'CPC Loss {loss_cpc.val:.6f} ({loss_cpc.local_avg:.4f})\t' 'Heading Loss {loss_hd.val:.6f} ({loss_hd.local_avg:.4f})\t' 'Acc: top1 {3:.4f}; top3 {4:.4f}; top5 {5:.4f} T:{6:.2f}\t'. format(epoch, idx, len(data_loader), top1_cpc, top3_cpc, top5_cpc, time.time() - tic, loss_cpc=losses_cpc, loss_hd=losses_hd)) writer_train.add_scalar('local/loss', losses_cpc.val, iteration) writer_train.add_scalar('local/accuracy', accuracy_cpc.val, iteration) iteration += 1 return losses_cpc.local_avg, accuracy_cpc.local_avg, [ i.local_avg for i in accuracy_list_cpc ], losses_hd.local_avg
def update_metrics(self, gt_all_features, flat_scores, stats, data): B, NS, N = self.args["batch_size"], self.args["pred_step"], self.args["num_seq"] loss_dict = {k: {} for k in self.losses} contexts = {} for mode in self.modes: SQ = self.models[mode].last_size ** 2 target_flattened = self.standard_grid_mask[mode].view(self.B0 * NS * SQ, self.B1 * NS * SQ) # CPC loss if True: score_flat = flat_scores[mode] target = target_flattened target_lbl = target.float().argmax(dim=1) # Compute and log performance metrics topKs = calc_topk_accuracy(score_flat, target_lbl, self.accuracyKList) for i in range(len(self.accuracyKList)): stats[mu.CPCLoss][mode]["acc" + str(self.accuracyKList[i])].update(topKs[i].item(), B) # Compute CPC loss for independent model training loss_dict[mu.CPCLoss][mode] = self.criterias[mu.CPCLoss](score_flat, target) for (m0, m1) in self.mode_pairs: tupName = self.get_tuple_name(m0, m1) # Cdot related losses if True: comp_gt_all0 = self.compiled_features[m0](gt_all_features[m0]).unsqueeze(3).unsqueeze(3) comp_gt_all1 = self.compiled_features[m1](gt_all_features[m1]).unsqueeze(3).unsqueeze(3) cdot0 = self.interModeDotHandler.get_cluster_dots(comp_gt_all0) cdot1 = self.interModeDotHandler.get_cluster_dots(comp_gt_all1) B, NS, B2, NS = cdot0.shape assert cdot0.shape == cdot1.shape == (B, NS, B2, NS), \ "Invalid shapes: {}, {}, {}".format(cdot0.shape, cdot1.shape, (B, NS, B2, NS)) cos_sim_dot_loss = self.criterias[self.CooperativeLossLabel](cdot0, cdot1) loss_dict[self.CooperativeLossLabel][tupName] = self.dot_wt * cos_sim_dot_loss # Modality sync loss if True: sync_loss, mode_stats = self.modeSyncers[tupName](gt_all_features[m0], gt_all_features[m1]) # stats: dict modeLoss -> specificStat for modeLoss in mode_stats.keys(): for stat in mode_stats[modeLoss].keys(): stats[modeLoss][tupName][stat].update(mode_stats[modeLoss][stat].item(), B) loss_dict[mu.ModeSim][tupName] = self.sync_wt * sync_loss return loss_dict, stats
def validate(data_loader, model, epoch, criterion): losses = AverageMeter() accuracy = AverageMeter() accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.eval() with torch.no_grad(): for idx, (input_seq, targets) in enumerate(data_loader): if args.dataset == 'tdw': if args.target == 'obj_categ': y_gt = targets['category'] elif args.target == 'self_motion': # m_tmp, theta_tmp = cart2pol(targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x'], # targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z']) # y_gt = torch.cat((m_tmp.unsqueeze(dim = 1), # theta_tmp.unsqueeze(dim = 1), # targets['camera_motion']['rotation']['yaw'].unsqueeze(dim = 1)), # dim = 1) # norm_cam = torch.linalg.norm(torch.cat( (100 * targets['camera_motion']['translation'] ['x_v'].unsqueeze(1), 100 * targets['camera_motion'] ['translation']['z_v'].unsqueeze(1)), dim=1), dim=1) norm_camobj = torch.linalg.norm(torch.cat( (targets['camera_object_vec']['x'].unsqueeze(1), targets['camera_object_vec']['z'].unsqueeze(1)), dim=1), dim=1) y_gt = torch.cat(( (100 * targets['camera_motion']['translation']['x_v'] / norm_cam - targets['camera_object_vec']['x'] / norm_camobj).unsqueeze(dim=1), (100 * targets['camera_motion']['translation']['z_v'] / norm_cam - targets['camera_object_vec']['z'] / norm_camobj).unsqueeze(dim=1), targets['camera_motion']['rotation']['yaw'].unsqueeze( dim=1)), dim=1) # # y_gt = torch.cat(((100*targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x']).unsqueeze(dim = 1), # (100*targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z']).unsqueeze(dim = 1) # ), # dim = 1) # # heading_speed, _ = cart2pol(targets['camera_motion']['translation']['x_v'], targets['camera_motion']['translation']['z_v']) # _, theta_1 = cart2pol(targets['camera_object_vec']['x'], targets['camera_object_vec']['z']) # _, theta_2 = cart2pol(targets['camera_motion']['translation']['x_v'],targets['camera_motion']['translation']['z_v']) # heading_theta = theta_1 - theta_2 # head_rotation_yaw = targets['camera_motion']['rotation']['yaw'] # y_gt = torch.cat(( # head_rotation_yaw.unsqueeze(dim = 1), # 100*heading_speed.unsqueeze(dim = 1) - 1.5), # dim = 1) # elif args.dataset == 'ucf101': y_gt = targets - 1 else: y_gt = targets tic = time.time() y_gt = y_gt.squeeze().to(cuda) input_seq = input_seq.squeeze().to(cuda) B = input_seq.size(0) y = model(input_seq) del input_seq loss = criterion(y, y_gt) if isinstance(criterion, nn.CrossEntropyLoss): top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2)) accuracy_list[0].update(top1.item(), B) accuracy.update(top1.item(), B) elif isinstance(criterion, nn.L1Loss) or isinstance( criterion, nn.MSELoss): loss = loss / B losses.update(loss.item(), B) if isinstance(criterion, nn.CrossEntropyLoss): print('[{0}/{1}] Loss {loss.local_avg:.4f}\t' 'Acc: top1 {2:.4f} \t'.format(epoch, args.epochs, *[i.avg for i in accuracy_list], loss=losses)) elif isinstance(criterion, nn.L1Loss) or isinstance(criterion, nn.MSELoss): print('[{0}/{1}] Loss {loss.local_avg:.4f}\t'.format(epoch, args.epochs, loss=losses)) return losses.local_avg, accuracy.local_avg, [ i.local_avg for i in accuracy_list ]
def train(data_loader, model, optimizer, epoch, criterion): losses = AverageMeter() accuracy = AverageMeter() accuracy_list = [AverageMeter(), AverageMeter(), AverageMeter()] model.train() global iteration for idx, (input_seq, targets) in enumerate(data_loader): # print(idx) if args.dataset == 'tdw': if args.target == 'obj_categ': y_gt = targets['category'] elif args.target == 'self_motion': # m_tmp, theta_tmp = cart2pol(targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x'], # targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z']) # y_gt = torch.cat((m_tmp.unsqueeze(dim = 1), # theta_tmp.unsqueeze(dim = 1), # targets['camera_motion']['rotation']['yaw'].unsqueeze(dim = 1)), # dim = 1) # norm_cam = torch.linalg.norm(torch.cat( (100 * targets['camera_motion']['translation'] ['x_v'].unsqueeze(1), 100 * targets['camera_motion'] ['translation']['z_v'].unsqueeze(1)), dim=1), dim=1) norm_camobj = torch.linalg.norm(torch.cat( (targets['camera_object_vec']['x'].unsqueeze(1), targets['camera_object_vec']['z'].unsqueeze(1)), dim=1), dim=1) y_gt = torch.cat( ((100 * targets['camera_motion']['translation']['x_v'] / norm_cam - targets['camera_object_vec']['x'] / norm_camobj).unsqueeze(dim=1), (100 * targets['camera_motion']['translation']['z_v'] / norm_cam - targets['camera_object_vec']['z'] / norm_camobj).unsqueeze(dim=1), targets['camera_motion']['rotation']['yaw'].unsqueeze( dim=1)), dim=1) # # y_gt = torch.cat(((100*targets['camera_motion']['translation']['x_v'] - targets['camera_object_vec']['x']).unsqueeze(dim = 1), # (100*targets['camera_motion']['translation']['z_v'] - targets['camera_object_vec']['z']).unsqueeze(dim = 1) # ), # dim = 1) # # heading_speed, _ = cart2pol(targets['camera_motion']['translation']['x_v'], targets['camera_motion']['translation']['z_v']) # _, theta_1 = cart2pol(targets['camera_object_vec']['x'], targets['camera_object_vec']['z']) # _, theta_2 = cart2pol(targets['camera_motion']['translation']['x_v'],targets['camera_motion']['translation']['z_v']) # heading_theta = theta_2 - theta_1 # head_rotation_yaw = targets['camera_motion']['rotation']['yaw'] # y_gt = torch.cat(( # head_rotation_yaw.unsqueeze(dim = 1), # 100*heading_speed.unsqueeze(dim = 1) - 1.5), # dim = 1) # elif args.dataset == 'ucf101': y_gt = targets - 1 else: y_gt = targets tic = time.time() y_gt = y_gt.squeeze().to(cuda) input_seq = input_seq.squeeze().to(cuda) B = input_seq.size(0) y = model(input_seq) del input_seq loss = criterion(y, y_gt) if isinstance(criterion, nn.CrossEntropyLoss): top1, top2 = calc_topk_accuracy(y, y_gt, (1, 2)) accuracy_list[0].update(top1.item(), B) accuracy.update(top1.item(), B) elif isinstance(criterion, nn.L1Loss) or isinstance( criterion, nn.MSELoss): loss = loss / B losses.update(loss.item(), B) optimizer.zero_grad() loss.backward() optimizer.step() del loss if idx % args.print_freq == 0: if isinstance(criterion, nn.CrossEntropyLoss): print('Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t' 'Acc: top1 {3:.4f}; T:{4:.2f}\t'.format(epoch, idx, len(data_loader), top1, time.time() - tic, loss=losses)) elif isinstance(criterion, nn.L1Loss) or isinstance( criterion, nn.MSELoss): print('Epoch: [{0}][{1}/{2}]\t' 'Loss {loss.val:.6f} ({loss.local_avg:.4f})\t' 'T:{3:.2f}\t'.format(epoch, idx, len(data_loader), time.time() - tic, loss=losses)) iteration += 1 return losses.local_avg, accuracy.local_avg, [ i.local_avg for i in accuracy_list ]