def eval(model_train_dict, model_pos, test_generator, train_generator_eval):
N = 0
epoch_loss_3d_valid = 0
epoch_loss_3d_train_eval = 0
with torch.no_grad():
model_pos.load_state_dict(model_train_dict)
model_pos.eval()
# Evaluate on test set
for cam, batch, batch_2d in test_generator.next_epoch():
inputs_3d = torch.from_numpy(batch.astype('float32'))
inputs_2d = torch.from_numpy(batch_2d.astype('float32'))
if torch.cuda.is_available():
inputs_3d = inputs_3d.cuda()
inputs_2d = inputs_2d.cuda()
inputs_3d[:, :, 0] = 0
# Predict 3D poses
predicted_3d_pos = model_pos(inputs_2d)
loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d)
epoch_loss_3d_valid += inputs_3d.shape[0] * inputs_3d.shape[
1] * loss_3d_pos.item()
N += inputs_3d.shape[0] * inputs_3d.shape[1]
losses_3d_valid_ave = epoch_loss_3d_valid / N
# Evaluate on training set, this time in evaluation mode
N = 0
for cam, batch, batch_2d in train_generator_eval.next_epoch():
if batch_2d.shape[1] == 0:
# This happens only when downsampling the dataset
continue
inputs_3d = torch.from_numpy(batch.astype('float32'))
inputs_2d = torch.from_numpy(batch_2d.astype('float32'))
if torch.cuda.is_available():
inputs_3d = inputs_3d.cuda()
inputs_2d = inputs_2d.cuda()
inputs_3d[:, :, 0] = 0
# Compute 3D poses
predicted_3d_pos = model_pos(inputs_2d)
loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d)
epoch_loss_3d_train_eval += inputs_3d.shape[0] * inputs_3d.shape[
1] * loss_3d_pos.item()
N += inputs_3d.shape[0] * inputs_3d.shape[1]
losses_3d_train_eval_ave = epoch_loss_3d_train_eval / N
return losses_3d_valid_ave, losses_3d_train_eval_ave
def train(model_pos_train, train_generator, optimizer):
epoch_loss_3d_train = 0
N = 0
# Regular supervised scenario
for _, batch_3d, batch_2d in train_generator.next_epoch():
inputs_3d = torch.from_numpy(batch_3d.astype('float32'))
inputs_2d = torch.from_numpy(batch_2d.astype('float32'))
if torch.cuda.is_available():
inputs_3d = inputs_3d.cuda()
inputs_2d = inputs_2d.cuda()
inputs_3d[:, :, 0] = 0
optimizer.zero_grad()
# Predict 3D poses
predicted_3d_pos = model_pos_train(inputs_2d)
loss_3d_pos = mpjpe(predicted_3d_pos, inputs_3d)
epoch_loss_3d_train += inputs_3d.shape[0] * inputs_3d.shape[1] * loss_3d_pos.item()
N += inputs_3d.shape[0] * inputs_3d.shape[1]
loss_total = loss_3d_pos
loss_total.backward()
optimizer.step()
epoch_losses_eva = epoch_loss_3d_train / N
return epoch_losses_eva
def forward(self, x, y):
assert len(
x.shape
) == 4, f'input shape should be (N T J 2), but got {x.shape}'
assert x.shape[-1] == 2
pred = self.tcn(x) # (N T J 3)
self.last_loss = mpjpe(pred, y)
self.last_output = pred
return pred
def evaluate(test_generator, model_pos, joints_left, joints_right, action=None, return_predictions=False):
epoch_loss_3d_pos = 0
epoch_loss_3d_pos_procrustes = 0
with torch.no_grad():
model_pos.eval()
N = 0
for _, batch, batch_2d in test_generator.next_epoch():
inputs_2d = torch.from_numpy(batch_2d.astype('float32'))
if torch.cuda.is_available():
inputs_2d = inputs_2d.cuda()
# Positional model
predicted_3d_pos = model_pos(inputs_2d)
# Test-time augmentation (if enabled)
if test_generator.augment_enabled():
# Undo flipping and take average with non-flipped version
predicted_3d_pos[1, :, :, 0] *= -1
predicted_3d_pos[1, :, joints_left + joints_right] = predicted_3d_pos[1, :, joints_right + joints_left]
predicted_3d_pos = torch.mean(predicted_3d_pos, dim=0, keepdim=True)
if return_predictions:
return predicted_3d_pos.squeeze(0).cpu().numpy()
inputs_3d = torch.from_numpy(batch.astype('float32'))
if torch.cuda.is_available():
inputs_3d = inputs_3d.cuda()
inputs_3d[:, :, 0] = 0
if test_generator.augment_enabled():
inputs_3d = inputs_3d[:1]
error = mpjpe(predicted_3d_pos, inputs_3d)
epoch_loss_3d_pos += inputs_3d.shape[0] * inputs_3d.shape[1] * error.item()
N += inputs_3d.shape[0] * inputs_3d.shape[1]
inputs = inputs_3d.cpu().numpy().reshape(-1, inputs_3d.shape[-2], inputs_3d.shape[-1])
predicted_3d_pos = predicted_3d_pos.cpu().numpy().reshape(-1, inputs_3d.shape[-2], inputs_3d.shape[-1])
epoch_loss_3d_pos_procrustes += inputs_3d.shape[0] * inputs_3d.shape[1] * p_mpjpe(predicted_3d_pos, inputs)
if action is None:
print('----------')
else:
print('----' + action + '----')
e1 = (epoch_loss_3d_pos / N) * 1000
e2 = (epoch_loss_3d_pos_procrustes / N) * 1000
print('Test time augmentation:', test_generator.augment_enabled())
print('Protocol #1 Error (MPJPE):', e1, 'mm')
print('Protocol #2 Error (P-MPJPE):', e2, 'mm')
print('----------')
return e1, e2
def evaluate(data_loader, model_pos, device, architecture):
batch_time = AverageMeter()
data_time = AverageMeter()
epoch_loss_3d_pos = AverageMeter()
epoch_loss_3d_pos_procrustes = AverageMeter()
predictions = []
# Switch to evaluate mode
torch.set_grad_enabled(False)
model_pos.eval()
end = time.time()
bar = Bar('Eval ', max=len(data_loader))
for i, (targets_3d, inputs_2d, _) in enumerate(data_loader):
# Measure data loading time
data_time.update(time.time() - end)
num_poses = targets_3d.size(0)
inputs_2d = inputs_2d.to(device)
if architecture == 'linear':
outputs_3d = model_pos(inputs_2d.view(num_poses,
-1)).view(num_poses, -1,
3).cpu()
outputs_3d = torch.cat(
[torch.zeros(num_poses, 1, outputs_3d.size(2)), outputs_3d],
1) # Pad hip joint
else:
outputs_3d = model_pos(inputs_2d).cpu()
outputs_3d[:, :, :] -= outputs_3d[:, :
1, :] # Zero-centre the root (hip)
predictions.append(outputs_3d.numpy())
epoch_loss_3d_pos.update(
mpjpe(outputs_3d, targets_3d).item() * 1000.0, num_poses)
epoch_loss_3d_pos_procrustes.update(
p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item() * 1000.0,
num_poses)
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \
'| MPJPE: {e1: .4f} | P-MPJPE: {e2: .4f}' \
.format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg,
ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_procrustes.avg)
bar.next()
bar.finish()
return np.concatenate(
predictions), epoch_loss_3d_pos.avg, epoch_loss_3d_pos_procrustes.avg
def prevalid(self):
from common.triangulation import triangulation_acc
from common.utils import AverageMeter
from common.loss import mpjpe
acc_pre = AverageMeter()
acc_best = AverageMeter()
acc_svd = AverageMeter()
for i in range(len(self)):
target_score, input, data_dict = self.__getitem__(i)
target_score = target_score.unsqueeze(0)
for item in data_dict.keys():
data_dict[item] = data_dict[item].unsqueeze(0)
output_3d_dict, targets_3d = triangulation_acc(target_score,
data_dict,
all_metric=True)
acc_pre.update(mpjpe(output_3d_dict['ltr_before'], targets_3d))
acc_best.update(mpjpe(output_3d_dict['ltr_best'], targets_3d))
acc_svd.update(mpjpe(output_3d_dict['ltr_svd'], targets_3d))
acc_str = "before: {:.5f}, best: {:.5f}, " \
"svd: {:.5f}".format(acc_pre.avg, acc_best.avg, acc_svd.avg)
print("Pre validation: ")
print(acc_str)
def forward(self, x, y):
assert len(
x.shape
) == 4, f'input shape should be (N T J 2), but got {x.shape}'
assert x.shape[-1] == 2
pred = self.tcn(x) # (N T J 3)
pred = pred.view(pred.shape[:2] + (-1, )).permute(0, 2, 1) # (N C T)
pred = self.nlb(pred) #(N C T)
pred = pred.permute(0, 2, 1).view(pred.shape[0], pred.shape[2], -1,
3) # (N T J 3)
self.last_loss = mpjpe(pred, y)
self.last_output = pred
return pred
def evaluate(data_loader, model_pos, device):
batch_time = AverageMeter()
data_time = AverageMeter()
epoch_loss_3d_pos = AverageMeter()
epoch_loss_3d_pos_procrustes = AverageMeter()
# Switch to evaluate mode
torch.set_grad_enabled(False)
model_pos.eval()
end = time.time()
bar = Bar('Eval ', max=len(data_loader))
for i, (targets_3d, inputs_2d, _) in enumerate(data_loader):
# Measure data loading time
data_time.update(time.time() - end)
num_poses = targets_3d.size(0)
inputs_2d = inputs_2d.to(device)
outputs_3d = model_pos(inputs_2d).cpu()
outputs_3d[:, :, :] -= outputs_3d[:, :
1, :] # Zero-centre the root (hip)
### remove scale
#inputs_2d = inputs_2d[:,:,:] - inputs_2d[:,:1,:]
#scale = compute_scale_pytorch(inputs_2d.cpu(), targets_3d)
#scale = torch.unsqueeze(torch.unsqueeze(scale, dim=1).expand(scale.size(0),17),dim=2)
#outputs_3d = outputs_3d*scale
###
epoch_loss_3d_pos.update(
mpjpe(outputs_3d, targets_3d).item() * 1000.0, num_poses)
epoch_loss_3d_pos_procrustes.update(
p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item() * 1000.0,
num_poses)
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \
'| MPJPE: {e1: .4f} | P-MPJPE: {e2: .4f}' \
.format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg,
ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_procrustes.avg)
bar.next()
bar.finish()
return epoch_loss_3d_pos.avg, epoch_loss_3d_pos_procrustes.avg
def evaluate(data_loader, model_pos, device):
batch_time = AverageMeter()
data_time = AverageMeter()
epoch_loss_3d_pos = AverageMeter()
epoch_loss_3d_pos_procrustes = AverageMeter()
# Switch to evaluate mode
torch.set_grad_enabled(False)
model_pos.eval()
end = time.time()
bar = Bar('Eval ', max=len(data_loader))
for i, (targets_3d, inputs_2d, _, intrinsics) in enumerate(data_loader):
# Measure data loading time
data_time.update(time.time() - end)
num_poses = targets_3d.size(0)
inputs_2d = inputs_2d.to(device)
outputs_3d = model_pos(inputs_2d).cpu()
output_2d = project_to_2d_linear(outputs_3d, intrinsics)
# true_output_2d = project_to_2d(targets_3d, intrinsics)
true_output_2d = project_to_2d_linear(targets_3d, intrinsics)
outputs_3d[:, :, :] -= outputs_3d[:, :
1, :] # Zero-centre the root (hip)
targets_3d[:, :, :] -= targets_3d[:, :
1, :] # Zero-centre the root (hip)
epoch_loss_3d_pos.update(
mpjpe(outputs_3d, targets_3d).item() * 1000.0, num_poses)
epoch_loss_3d_pos_procrustes.update(
p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item() * 1000.0,
num_poses)
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \
'| MPJPE: {e1: .4f} | P-MPJPE: {e2: .4f}' \
.format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg,
ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_procrustes.avg)
bar.next()
bar.finish()
return epoch_loss_3d_pos.avg, epoch_loss_3d_pos_procrustes.avg
def evaluate(data_loader, model_pos, device):
batch_time = AverageMeter()
data_time = AverageMeter()
epoch_loss_3d_pos = AverageMeter()
# epoch_loss_3d_pos_procrustes = AverageMeter()
epoch_loss_3d_pos_relative = AverageMeter()
# Switch to evaluate mode
torch.set_grad_enabled(False)
model_pos.eval()
end = time.time()
bar = Bar('Eval ', max=len(data_loader))
for i, (targets_score, inputs_2d, data_dict) in enumerate(data_loader):
# Measure data loading time
data_time.update(time.time() - end)
num_poses = targets_score.size(0)
inputs_2d = inputs_2d.to(device)
outputs_score = model_pos(inputs_2d).cpu()
# outputs_3d[:, :, :] -= outputs_3d[:, :1, :] # Zero-centre the root (hip)
output_3d, targets_3d = triangulation_acc(outputs_score, data_dict=data_dict, all_metric=False)
epoch_loss_3d_pos.update(mpjpe(output_3d['ltr_after'], targets_3d).item(), num_poses)
epoch_loss_3d_pos_relative.update(rel_mpjpe(output_3d['ltr_after'], targets_3d).item(), num_poses)
# epoch_loss_3d_pos_procrustes.update(p_mpjpe(outputs_3d.numpy(), targets_3d.numpy()).item(), num_poses)
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \
'| MPJPE: {e1: .8f} | REL-MPJPE: {e2: .8f}' \
.format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg,
ttl=bar.elapsed_td, eta=bar.eta_td, e1=epoch_loss_3d_pos.avg, e2=epoch_loss_3d_pos_relative.avg)
bar.next()
bar.finish()
return epoch_loss_3d_pos.avg, epoch_loss_3d_pos_relative.avg
def train(loss_last, data_loader, model_pos, criterion, optimizer, device, lr_init, lr_now, step, decay, gamma,
max_norm=True, loss_3d=False, earlyend=True):
batch_time = AverageMeter()
data_time = AverageMeter()
epoch_loss = AverageMeter()
# Switch to train mode
torch.set_grad_enabled(True)
model_pos.train()
end = time.time()
dataperepoch_limit = 1e10
dataperepoch_count = 0
if earlyend:
dataperepoch_limit = 1
bar = Bar('Train', max=len(data_loader))
for i, (targets_score, inputs_2d, data_dict) in enumerate(data_loader):
# Measure data loading time
data_time.update(time.time() - end)
num_poses = targets_score.size(0)
dataperepoch_count += num_poses
if dataperepoch_count >= dataperepoch_limit:
break
step += 1
if step % decay == 0 or step == 1:
lr_now = lr_decay(optimizer, step, lr_init, decay, gamma)
targets_score, inputs_2d = targets_score.to(device), inputs_2d.to(device)
outputs_score = model_pos(inputs_2d)
optimizer.zero_grad()
### 3d loss
if loss_last < 0.3 or loss_3d:
loss_3d = True
for key in data_dict.keys():
if isinstance(data_dict[key], torch.Tensor):
data_dict[key] = data_dict[key].to(device)
output_3d, targets_3d = triangulation_acc(outputs_score, data_dict=data_dict, all_metric=False)
loss_ = mpjpe(output_3d['ltr_after'], targets_3d)
else:
loss_ = criterion(outputs_score, targets_score)
loss_.backward()
if max_norm:
nn.utils.clip_grad_norm_(model_pos.parameters(), max_norm=1)
optimizer.step()
epoch_loss.update(loss_.item(), num_poses)
# Measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
bar.suffix = '({batch}/{size}) Data: {data:.6f}s | Batch: {bt:.3f}s | Total: {ttl:} | ETA: {eta:} ' \
'| Loss: {loss: .4f}' \
.format(batch=i + 1, size=len(data_loader), data=data_time.val, bt=batch_time.avg,
ttl=bar.elapsed_td, eta=bar.eta_td, loss=epoch_loss.avg)
bar.next()
bar.finish()
return epoch_loss.avg, lr_now, step
