def eval(model_train_dict, model_pos, test_generator, train_generator_eval): N = 0 epoch_loss_3d_valid = 0 epoch_loss_3d_train_eval = 0 with torch.no_grad(): model_pos.load_state_dict(model_train_dict) model_pos.eval() # Evaluate on test set for cam, batch, batch_2d in test_generator.next_epoch(): inputs_3d = torch.from_numpy(batch.astype('float32')) inputs_2d = torch.from_numpy(batch_2d.astype('float32')) if torch.cuda.is_available(): inputs_3d = inputs_3d.cuda() inputs_2d = inputs_2d.cuda() inputs_3d[:, :, 0] = 0 # Predict 3D poses predicted_3d_pos = model_pos(inputs_2d) loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d) epoch_loss_3d_valid += inputs_3d.shape[0] * inputs_3d.shape[ 1] * loss_3d_pos.item() N += inputs_3d.shape[0] * inputs_3d.shape[1] losses_3d_valid_ave = epoch_loss_3d_valid / N # Evaluate on training set, this time in evaluation mode N = 0 for cam, batch, batch_2d in train_generator_eval.next_epoch(): if batch_2d.shape[1] == 0: # This happens only when downsampling the dataset continue inputs_3d = torch.from_numpy(batch.astype('float32')) inputs_2d = torch.from_numpy(batch_2d.astype('float32')) if torch.cuda.is_available(): inputs_3d = inputs_3d.cuda() inputs_2d = inputs_2d.cuda() inputs_3d[:, :, 0] = 0 # Compute 3D poses predicted_3d_pos = model_pos(inputs_2d) loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d) epoch_loss_3d_train_eval += inputs_3d.shape[0] * inputs_3d.shape[ 1] * loss_3d_pos.item() N += inputs_3d.shape[0] * inputs_3d.shape[1] losses_3d_train_eval_ave = epoch_loss_3d_train_eval / N return losses_3d_valid_ave, losses_3d_train_eval_ave
def train(model_pos_train, train_generator, optimizer): epoch_loss_3d_train = 0 N = 0 # Regular supervised scenario for _, batch_3d, batch_2d in train_generator.next_epoch(): inputs_3d = torch.from_numpy(batch_3d.astype('float32')) inputs_2d = torch.from_numpy(batch_2d.astype('float32')) if torch.cuda.is_available(): inputs_3d = inputs_3d.cuda() inputs_2d = inputs_2d.cuda() inputs_3d[:, :, 0] = 0 optimizer.zero_grad() # Predict 3D poses predicted_3d_pos = model_pos_train(inputs_2d) loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d) epoch_loss_3d_train += inputs_3d.shape[0] * inputs_3d.shape[1] * loss_3d_pos.item() N += inputs_3d.shape[0] * inputs_3d.shape[1] loss_total = loss_3d_pos loss_total.backward() optimizer.step() epoch_losses_eva = epoch_loss_3d_train / N return epoch_losses_eva
def forward(self, x, y): assert len( x.shape ) == 4, f'input shape should be (N T J 2), but got {x.shape}' assert x.shape[-1] == 2 pred = self.tcn(x) # (N T J 3) self.last_loss = mpjpe(pred, y) self.last_output = pred return pred
def evaluate(test_generator, model_pos, joints_left, joints_right, action=None, return_predictions=False): epoch_loss_3d_pos = 0 epoch_loss_3d_pos_procrustes = 0 with torch.no_grad(): model_pos.eval() N = 0 for _, batch, batch_2d in test_generator.next_epoch(): inputs_2d = torch.from_numpy(batch_2d.astype('float32')) if torch.cuda.is_available(): inputs_2d = inputs_2d.cuda() # Positional model predicted_3d_pos = model_pos(inputs_2d) # Test-time augmentation (if enabled) if test_generator.augment_enabled(): # Undo flipping and take average with non-flipped version predicted_3d_pos[1, :, :, 0] *= -1 predicted_3d_pos[1, :, joints_left + joints_right] = predicted_3d_pos[1, :, joints_right + joints_left] predicted_3d_pos = torch.mean(predicted_3d_pos, dim=0, keepdim=True) if return_predictions: return predicted_3d_pos.squeeze(0).cpu().numpy() inputs_3d = torch.from_numpy(batch.astype('float32')) if torch.cuda.is_available(): inputs_3d = inputs_3d.cuda() inputs_3d[:, :, 0] = 0 if test_generator.augment_enabled(): inputs_3d = inputs_3d[:1] error = mpjpe(predicted_3d_pos, inputs_3d) epoch_loss_3d_pos += inputs_3d.shape[0] * inputs_3d.shape[1] * error.item() N += inputs_3d.shape[0] * inputs_3d.shape[1] inputs = inputs_3d.cpu().numpy().reshape(-1, inputs_3d.shape[-2], inputs_3d.shape[-1]) predicted_3d_pos = predicted_3d_pos.cpu().numpy().reshape(-1, inputs_3d.shape[-2], inputs_3d.shape[-1]) epoch_loss_3d_pos_procrustes += inputs_3d.shape[0] * inputs_3d.shape[1] * p_mpjpe(predicted_3d_pos, inputs) if action is None: print('----------') else: print('----' + action + '----') e1 = (epoch_loss_3d_pos / N) * 1000 e2 = (epoch_loss_3d_pos_procrustes / N) * 1000 print('Test time augmentation:', test_generator.augment_enabled()) print('Protocol #1 Error (MPJPE):', e1, 'mm') print('Protocol #2 Error (P-MPJPE):', e2, 'mm') print('----------') return e1, e2
def evaluate(data_loader, model_pos, device, architecture): batch_time = AverageMeter() data_time = AverageMeter() epoch_loss_3d_pos = AverageMeter() epoch_loss_3d_pos_procrustes = AverageMeter() predictions = [] # Switch to evaluate mode torch.set_grad_enabled(False) model_pos.eval() end = time.time() bar = Bar('Eval ', max=len(data_loader)) for i, (targets_3d, inputs_2d, _) in enumerate(data_loader): # Measure data loading time data_time.update(time.time() - end) num_poses = targets_3d.size(0) inputs_2d = inputs_2d.to(device) if architecture == 'linear': outputs_3d = model_pos(inputs_2d.view(num_poses, -1)).view(num_poses, -1, 3).cpu() outputs_3d = torch.cat( [torch.zeros(num_poses, 1, outputs_3d.size(2)), outputs_3d], 1) # Pad hip joint else: outputs_3d = model_pos(inputs_2d).cpu() outputs_3d[:, :, :] -= outputs_3d[:, : 1, :] # Zero-centre the root (hip) predictions.append(outputs_3d.numpy()) epoch_loss_3d_pos.update( mpjpe(outputs_3d, targets_3d).item() * 1000.0, num_poses) epoch_loss_3d_pos_procrustes.update( p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item() * 1000.0, num_poses) # Measure elapsed time batch_time.update(time.time() - end) end = time.time() bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \ '| MPJPE: {e1: .4f} | P-MPJPE: {e2: .4f}' \ .format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg, ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_procrustes.avg) bar.next() bar.finish() return np.concatenate( predictions), epoch_loss_3d_pos.avg, epoch_loss_3d_pos_procrustes.avg
def prevalid(self): from common.triangulation import triangulation_acc from common.utils import AverageMeter from common.loss import mpjpe acc_pre = AverageMeter() acc_best = AverageMeter() acc_svd = AverageMeter() for i in range(len(self)): target_score, input, data_dict = self.__getitem__(i) target_score = target_score.unsqueeze(0) for item in data_dict.keys(): data_dict[item] = data_dict[item].unsqueeze(0) output_3d_dict, targets_3d = triangulation_acc(target_score, data_dict, all_metric=True) acc_pre.update(mpjpe(output_3d_dict['ltr_before'], targets_3d)) acc_best.update(mpjpe(output_3d_dict['ltr_best'], targets_3d)) acc_svd.update(mpjpe(output_3d_dict['ltr_svd'], targets_3d)) acc_str = "before: {:.5f}, best: {:.5f}, " \ "svd: {:.5f}".format(acc_pre.avg, acc_best.avg, acc_svd.avg) print("Pre validation: ") print(acc_str)
def forward(self, x, y): assert len( x.shape ) == 4, f'input shape should be (N T J 2), but got {x.shape}' assert x.shape[-1] == 2 pred = self.tcn(x) # (N T J 3) pred = pred.view(pred.shape[:2] + (-1, )).permute(0, 2, 1) # (N C T) pred = self.nlb(pred) #(N C T) pred = pred.permute(0, 2, 1).view(pred.shape[0], pred.shape[2], -1, 3) # (N T J 3) self.last_loss = mpjpe(pred, y) self.last_output = pred return pred
def evaluate(data_loader, model_pos, device): batch_time = AverageMeter() data_time = AverageMeter() epoch_loss_3d_pos = AverageMeter() epoch_loss_3d_pos_procrustes = AverageMeter() # Switch to evaluate mode torch.set_grad_enabled(False) model_pos.eval() end = time.time() bar = Bar('Eval ', max=len(data_loader)) for i, (targets_3d, inputs_2d, _) in enumerate(data_loader): # Measure data loading time data_time.update(time.time() - end) num_poses = targets_3d.size(0) inputs_2d = inputs_2d.to(device) outputs_3d = model_pos(inputs_2d).cpu() outputs_3d[:, :, :] -= outputs_3d[:, : 1, :] # Zero-centre the root (hip) ### remove scale #inputs_2d = inputs_2d[:,:,:] - inputs_2d[:,:1,:] #scale = compute_scale_pytorch(inputs_2d.cpu(), targets_3d) #scale = torch.unsqueeze(torch.unsqueeze(scale, dim=1).expand(scale.size(0),17),dim=2) #outputs_3d = outputs_3d*scale ### epoch_loss_3d_pos.update( mpjpe(outputs_3d, targets_3d).item() * 1000.0, num_poses) epoch_loss_3d_pos_procrustes.update( p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item() * 1000.0, num_poses) # Measure elapsed time batch_time.update(time.time() - end) end = time.time() bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \ '| MPJPE: {e1: .4f} | P-MPJPE: {e2: .4f}' \ .format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg, ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_procrustes.avg) bar.next() bar.finish() return epoch_loss_3d_pos.avg, epoch_loss_3d_pos_procrustes.avg
def evaluate(data_loader, model_pos, device): batch_time = AverageMeter() data_time = AverageMeter() epoch_loss_3d_pos = AverageMeter() epoch_loss_3d_pos_procrustes = AverageMeter() # Switch to evaluate mode torch.set_grad_enabled(False) model_pos.eval() end = time.time() bar = Bar('Eval ', max=len(data_loader)) for i, (targets_3d, inputs_2d, _, intrinsics) in enumerate(data_loader): # Measure data loading time data_time.update(time.time() - end) num_poses = targets_3d.size(0) inputs_2d = inputs_2d.to(device) outputs_3d = model_pos(inputs_2d).cpu() output_2d = project_to_2d_linear(outputs_3d, intrinsics) # true_output_2d = project_to_2d(targets_3d, intrinsics) true_output_2d = project_to_2d_linear(targets_3d, intrinsics) outputs_3d[:, :, :] -= outputs_3d[:, : 1, :] # Zero-centre the root (hip) targets_3d[:, :, :] -= targets_3d[:, : 1, :] # Zero-centre the root (hip) epoch_loss_3d_pos.update( mpjpe(outputs_3d, targets_3d).item() * 1000.0, num_poses) epoch_loss_3d_pos_procrustes.update( p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item() * 1000.0, num_poses) # Measure elapsed time batch_time.update(time.time() - end) end = time.time() bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \ '| MPJPE: {e1: .4f} | P-MPJPE: {e2: .4f}' \ .format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg, ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_procrustes.avg) bar.next() bar.finish() return epoch_loss_3d_pos.avg, epoch_loss_3d_pos_procrustes.avg
def evaluate(data_loader, model_pos, device): batch_time = AverageMeter() data_time = AverageMeter() epoch_loss_3d_pos = AverageMeter() # epoch_loss_3d_pos_procrustes = AverageMeter() epoch_loss_3d_pos_relative = AverageMeter() # Switch to evaluate mode torch.set_grad_enabled(False) model_pos.eval() end = time.time() bar = Bar('Eval ', max=len(data_loader)) for i, (targets_score, inputs_2d, data_dict) in enumerate(data_loader): # Measure data loading time data_time.update(time.time() - end) num_poses = targets_score.size(0) inputs_2d = inputs_2d.to(device) outputs_score = model_pos(inputs_2d).cpu() # outputs_3d[:, :, :] -= outputs_3d[:, :1, :] # Zero-centre the root (hip) output_3d, targets_3d = triangulation_acc(outputs_score, data_dict=data_dict, all_metric=False) epoch_loss_3d_pos.update(mpjpe(output_3d['ltr_after'], targets_3d).item(), num_poses) epoch_loss_3d_pos_relative.update(rel_mpjpe(output_3d['ltr_after'], targets_3d).item(), num_poses) # epoch_loss_3d_pos_procrustes.update(p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item(), num_poses) # Measure elapsed time batch_time.update(time.time() - end) end = time.time() bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \ '| MPJPE: {e1: .8f} | REL-MPJPE: {e2: .8f}' \ .format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg, ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_relative.avg) bar.next() bar.finish() return epoch_loss_3d_pos.avg, epoch_loss_3d_pos_relative.avg
def train(loss_last, data_loader, model_pos, criterion, optimizer, device, lr_init, lr_now, step, decay, gamma, max_norm=True, loss_3d=False, earlyend=True): batch_time = AverageMeter() data_time = AverageMeter() epoch_loss = AverageMeter() # Switch to train mode torch.set_grad_enabled(True) model_pos.train() end = time.time() dataperepoch_limit = 1e10 dataperepoch_count = 0 if earlyend: dataperepoch_limit = 1 bar = Bar('Train', max=len(data_loader)) for i, (targets_score, inputs_2d, data_dict) in enumerate(data_loader): # Measure data loading time data_time.update(time.time() - end) num_poses = targets_score.size(0) dataperepoch_count += num_poses if dataperepoch_count >= dataperepoch_limit: break step += 1 if step % decay == 0 or step == 1: lr_now = lr_decay(optimizer, step, lr_init, decay, gamma) targets_score, inputs_2d = targets_score.to(device), inputs_2d.to(device) outputs_score = model_pos(inputs_2d) optimizer.zero_grad() ### 3d loss if loss_last < 0.3 or loss_3d: loss_3d = True for key in data_dict.keys(): if isinstance(data_dict[key], torch.Tensor): data_dict[key] = data_dict[key].to(device) output_3d, targets_3d = triangulation_acc(outputs_score, data_dict=data_dict, all_metric=False) loss_ = mpjpe(output_3d['ltr_after'], targets_3d) else: loss_ = criterion(outputs_score, targets_score) loss_.backward() if max_norm: nn.utils.clip_grad_norm_(model_pos.parameters(), max_norm=1) optimizer.step() epoch_loss.update(loss_.item(), num_poses) # Measure elapsed time batch_time.update(time.time() - end) end = time.time() bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \ '| Loss: {loss: .4f}' \ .format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg, ttl=bar.elapsed_td, eta=bar.eta_td, loss=epoch_loss.avg) bar.next() bar.finish() return epoch_loss.avg, lr_now, step