def validate(self):
self.model.eval()
start = time.time()
summary_string = ''
bar = Bar('Validation', fill='#', max=len(self.test_loader))
if self.evaluation_accumulators is not None:
for k,v in self.evaluation_accumulators.items():
self.evaluation_accumulators[k] = []
J_regressor = torch.from_numpy(np.load(osp.join(VIBE_DATA_DIR, 'J_regressor_h36m.npy'))).float()
for i, target in enumerate(self.test_loader):
# video = video.to(self.device)
move_dict_to_device(target, self.device)
# <=============
with torch.no_grad():
#inp = target['features']
imgIn = target['video']
# imgIn = imgIn.to('cuda:0')
preds = self.model(imgIn, J_regressor=J_regressor)
#preds = self.model(inp, J_regressor=J_regressor)
# convert to 14 keypoint format for evaluation
# if self.use_spin:
n_kp = preds[-1]['kp_3d'].shape[-2]
pred_j3d = preds[-1]['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
target_j3d = target['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
pred_verts = preds[-1]['verts'].view(-1, 6890, 3).cpu().numpy()
target_theta = target['theta'].view(-1, 85).cpu().numpy()
self.evaluation_accumulators['pred_verts'].append(pred_verts)
self.evaluation_accumulators['target_theta'].append(target_theta)
self.evaluation_accumulators['pred_j3d'].append(pred_j3d)
self.evaluation_accumulators['target_j3d'].append(target_j3d)
# =============>
batch_time = time.time() - start
summary_string = f'({i + 1}/{len(self.test_loader)}) | batch: {batch_time * 10.0:.4}ms | ' \
f'Total: {bar.elapsed_td} | ETA: {bar.eta_td:}'
bar.suffix = summary_string
bar.next()
bar.finish()
logger.info(summary_string)
def validate(self):
self.generator.eval()
start = time.time()
summary_string = ''
bar = Bar('Validation', fill='#', max=len(self.valid_loader))
if self.evaluation_accumulators is not None:
for k, v in self.evaluation_accumulators.items():
self.evaluation_accumulators[k] = []
J_regressor = torch.from_numpy(
np.load(osp.join(VIBE_DATA_DIR, 'J_regressor_h36m.npy'))).float()
for i, target in enumerate(self.valid_loader):
move_dict_to_device(target, self.device)
# <=============
with torch.no_grad():
inp = target['features']
preds = self.generator(inp, J_regressor=J_regressor)
# convert to 14 keypoint format for evaluation
n_kp = preds[-1]['kp_3d'].shape[-2]
pred_j3d = preds[-1]['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
target_j3d = target['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
pred_verts = preds[-1]['verts'].view(-1, 6890, 3).cpu().numpy()
target_theta = target['theta'].view(-1, 85).cpu().numpy()
self.evaluation_accumulators['pred_verts'].append(pred_verts)
self.evaluation_accumulators['target_theta'].append(
target_theta)
self.evaluation_accumulators['pred_j3d'].append(pred_j3d)
self.evaluation_accumulators['target_j3d'].append(target_j3d)
# =============>
# <============= DEBUG
if self.debug and self.valid_global_step % self.debug_freq == 0:
from lib.utils.vis import batch_visualize_vid_preds
video = target['video']
dataset = 'common'
vid_tensor = batch_visualize_vid_preds(video,
preds[-1],
target,
vis_hmr=False,
dataset=dataset)
self.writer.add_video('valid-video',
vid_tensor,
global_step=self.valid_global_step,
fps=10)
# =============>
batch_time = time.time() - start
summary_string = f'({i + 1}/{len(self.valid_loader)}) | batch: {batch_time * 10.0:.4}ms | ' \
f'Total: {bar.elapsed_td} | ETA: {bar.eta_td:}'
self.valid_global_step += 1
bar.suffix = summary_string
bar.next()
bar.finish()
logger.info(summary_string)
def train(self):
# Single epoch training routine
losses = AverageMeter()
timer = {
'data': 0,
'forward': 0,
'loss': 0,
'backward': 0,
'batch': 0,
}
self.generator.train()
self.motion_discriminator.train()
start = time.time()
summary_string = ''
bar = Bar(f'Epoch {self.epoch + 1}/{self.end_epoch}',
fill='#',
max=self.num_iters_per_epoch)
for i in range(self.num_iters_per_epoch):
# Dirty solution to reset an iterator
target_2d = target_3d = None
if self.train_2d_iter:
try:
target_2d = next(self.train_2d_iter)
except StopIteration:
self.train_2d_iter = iter(self.train_2d_loader)
target_2d = next(self.train_2d_iter)
move_dict_to_device(target_2d, self.device)
if self.train_3d_iter:
try:
target_3d = next(self.train_3d_iter)
except StopIteration:
self.train_3d_iter = iter(self.train_3d_loader)
target_3d = next(self.train_3d_iter)
move_dict_to_device(target_3d, self.device)
real_body_samples = real_motion_samples = None
try:
real_motion_samples = next(self.disc_motion_iter)
except StopIteration:
self.disc_motion_iter = iter(self.disc_motion_loader)
real_motion_samples = next(self.disc_motion_iter)
move_dict_to_device(real_motion_samples, self.device)
# <======= Feedforward generator and discriminator
if target_2d and target_3d:
inp = torch.cat((target_2d['features'], target_3d['features']),
dim=0).to(self.device)
elif target_3d:
inp = target_3d['features'].to(self.device)
else:
inp = target_2d['features'].to(self.device)
timer['data'] = time.time() - start
start = time.time()
preds = self.generator(inp)
timer['forward'] = time.time() - start
start = time.time()
gen_loss, motion_dis_loss, loss_dict = self.criterion(
generator_outputs=preds,
data_2d=target_2d,
data_3d=target_3d,
data_body_mosh=real_body_samples,
data_motion_mosh=real_motion_samples,
motion_discriminator=self.motion_discriminator,
)
# =======>
timer['loss'] = time.time() - start
start = time.time()
# <======= Backprop generator and discriminator
self.gen_optimizer.zero_grad()
gen_loss.backward()
self.gen_optimizer.step()
if self.train_global_step % self.dis_motion_update_steps == 0:
self.dis_motion_optimizer.zero_grad()
motion_dis_loss.backward()
self.dis_motion_optimizer.step()
# =======>
# <======= Log training info
total_loss = gen_loss + motion_dis_loss
losses.update(total_loss.item(), inp.size(0))
timer['backward'] = time.time() - start
timer['batch'] = timer['data'] + timer['forward'] + timer[
'loss'] + timer['backward']
start = time.time()
summary_string = f'({i + 1}/{self.num_iters_per_epoch}) | Total: {bar.elapsed_td} | ' \
f'ETA: {bar.eta_td:} | loss: {losses.avg:.4f}'
for k, v in loss_dict.items():
summary_string += f' | {k}: {v:.2f}'
self.writer.add_scalar('train_loss/' + k,
v,
global_step=self.train_global_step)
for k, v in timer.items():
summary_string += f' | {k}: {v:.2f}'
self.writer.add_scalar('train_loss/loss',
total_loss.item(),
global_step=self.train_global_step)
if self.debug:
print('==== Visualize ====')
from lib.utils.vis import batch_visualize_vid_preds
video = target_3d['video']
dataset = 'spin'
vid_tensor = batch_visualize_vid_preds(video,
preds[-1],
target_3d.copy(),
vis_hmr=False,
dataset=dataset)
self.writer.add_video('train-video',
vid_tensor,
global_step=self.train_global_step,
fps=10)
self.train_global_step += 1
bar.suffix = summary_string
bar.next()
if torch.isnan(total_loss):
exit('Nan value in loss, exiting!...')
# =======>
bar.finish()
logger.info(summary_string)
def validate(self):
self.model.eval()
start = time.time()
summary_string = ''
bar = Bar('Validation', fill='#', max=len(self.test_loader))
if self.evaluation_accumulators is not None:
for k, v in self.evaluation_accumulators.items():
self.evaluation_accumulators[k] = []
J_regressor = torch.from_numpy(
np.load(osp.join(VIBE_DATA_DIR, 'J_regressor_h36m.npy'))).float()
preds_acc = []
targets_acc = []
for i, target in enumerate(self.test_loader):
# video = video.to(self.device)
move_dict_to_device(target, self.device)
# <=============
with torch.no_grad():
inp = target['features']
# preds = self.model(inp, J_regressor=J_regressor, refiner = self.refiner)
preds = self.model(inp, J_regressor=J_regressor)
# convert to 14 keypoint format for evaluation
# if self.use_spin:
n_kp = preds[-1]['kp_3d'].shape[-2]
pred_j3d = preds[-1]['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
target_j3d = target['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
pred_verts = preds[-1]['verts'].view(-1, 6890, 3).cpu().numpy()
target_theta = target['theta'].view(-1, 85).cpu().numpy()
self.evaluation_accumulators['pred_verts'].append(pred_verts)
self.evaluation_accumulators['target_theta'].append(
target_theta)
self.evaluation_accumulators['pred_j3d'].append(pred_j3d)
self.evaluation_accumulators['target_j3d'].append(target_j3d)
######################## Data saver #####################
preds_acc.append({k:v.cpu().numpy() if torch.is_tensor(v) else v for k, v in preds[-1].items() \
if k in ['features', 'theta', 'instance_id']})
targets_acc.append({k:v.cpu().numpy() if torch.is_tensor(v) else v for k, v in target.items() \
if k in ['features', 'theta', 'instance_id']})
######################## vis #####################
# renderer = SMPL_M_Renderer(render_size = (400, 400))
# mocap_pose = vertizalize_smpl_root(torch.tensor(target_theta[:100,3:75]))
# meva_pose = vertizalize_smpl_root(preds[-1]['theta'].view(-1, 85)[:100,3:75]).cpu().numpy()
# mocap_images = renderer.render_smpl(mocap_pose)
# meva_images = renderer.render_smpl(meva_pose)
# videos = [mocap_images, meva_images]
# grid_size = [1,len(videos)]
# descriptions = ["Mocap", "VIBE", "MEVA"]
# output_name = "{}/output_meva_d{:02d}.mp4".format("/hdd/zen/data/ActmixGenenerator/output/3dpw/mevav2", 0)
# print(output_name)
# assemble_videos(videos, grid_size, descriptions, output_name)
######################## vis #####################
del target, preds
torch.cuda.empty_cache()
# =============>
batch_time = time.time() - start
summary_string = f'({i + 1}/{len(self.test_loader)}) | batch: {batch_time * 10.0:.4}ms | ' \
f'Total: {bar.elapsed_td} | ETA: {bar.eta_td:}'
bar.suffix = summary_string
bar.next()
bar.finish()
logger.info(summary_string)
meva_3dpw_res = {
"preds": preds_acc,
"targets": targets_acc,
}
pk.dump(meva_3dpw_res, open("3dpw_meva_res.pkl", "wb"))
def train(self):
# Single epoch training routine
losses = AverageMeter()
timer = {
'data': 0,
'forward': 0,
'loss': 0,
'backward': 0,
'batch': 0,
}
self.generator.train()
start = time.time()
summary_string = ''
bar = Bar(f'Epoch {self.epoch + 1}/{self.end_epoch}',
fill='#',
max=self.num_iters_per_epoch)
for i in range(self.num_iters_per_epoch):
# Dirty solution to reset an iterator
target_3dpw = None
if self.train_3dpw_iter:
try:
target_3dpw = next(self.train_3dpw_iter)
except StopIteration:
self.train_3dpw_iter = iter(self.valid_loader)
target_3dpw = next(self.train_3dpw_iter)
move_dict_to_device(target_3dpw, self.device)
timer['data'] = time.time() - start
start = time.time()
inp = target_3dpw['features']
preds = self.generator(inp) #dict = {’pose‘:}
timer['forward'] = time.time() - start
start = time.time()
gen_loss = self.criterion(generator_outputs=preds,
real_pose=target_3dpw['pose'],
real_shape=target_3dpw['shape'],
real_joints3D=target_3dpw['joints3D'])
# =======>
timer['loss'] = time.time() - start
start = time.time()
# <======= Backprop generator and discriminator
self.gen_optimizer.zero_grad()
gen_loss.backward()
self.gen_optimizer.step()
# <======= Log training info
total_loss = gen_loss
losses.update(total_loss.item(), inp.size(0))
timer['backward'] = time.time() - start
timer['batch'] = timer['data'] + timer['forward'] + timer[
'loss'] + timer['backward']
start = time.time()
summary_string = f'({i + 1}/{self.num_iters_per_epoch}) | Total: {bar.elapsed_td} | ' \
f'ETA: {bar.eta_td:} | loss: {losses.avg:.4f}'
for k, v in timer.items():
summary_string += f' | {k}: {v:.2f}'
bar.suffix = summary_string
bar.next()
if torch.isnan(total_loss):
exit('Nan value in loss, exiting!...')
# =======>
bar.finish()
logger.info(summary_string)
def validate(self):
self.model.eval()
start = time.time()
summary_string = ''
bar = Bar('Validation', fill='#', max=len(self.test_loader))
if self.evaluation_accumulators is not None:
for k,v in self.evaluation_accumulators.items():
self.evaluation_accumulators[k] = []
J_regressor = torch.from_numpy(np.load(osp.join(VIBE_DATA_DIR, 'J_regressor_h36m.npy'))).float()
for i, target in enumerate(self.test_loader):
# video = video.to(self.device)
move_dict_to_device(target, self.device)
# <=============
with torch.no_grad():
inp = target['features']
# preds = self.model(inp, J_regressor=J_regressor, refiner = self.refiner)
preds = self.model(inp, J_regressor=J_regressor)
# convert to 14 keypoint format for evaluation
# if self.use_spin:
n_kp = preds[-1]['kp_3d'].shape[-2]
pred_j3d = preds[-1]['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
target_j3d = target['kp_3d'].view(-1, n_kp, 3).cpu().numpy()
pred_verts = preds[-1]['verts'].view(-1, 6890, 3).cpu().numpy()
target_theta = target['theta'].view(-1, 85).cpu().numpy()
######################## vis #####################
# renderer = smpl_renderer.SMPL_Renderer(image_size = 400, camera_mode="look_at")
# target_pose = target_theta[:,3:75]
# pred_pose = preds[-1]['theta'][:,:,3:75].squeeze()
# renderer.render_pose_vid(torch.tensor(target_pose), out_file_name = "output/gt{:02d}.mp4".format(i), random_camera = 2, random_shape=False)
# renderer.render_pose_vid(torch.tensor(pred_pose), out_file_name = "output/ref{:02d}.mp4".format(i), random_camera = 2, random_shape=False)
######################## vis #####################
self.evaluation_accumulators['pred_verts'].append(pred_verts)
self.evaluation_accumulators['target_theta'].append(target_theta)
self.evaluation_accumulators['pred_j3d'].append(pred_j3d)
self.evaluation_accumulators['target_j3d'].append(target_j3d)
del target, preds
torch.cuda.empty_cache()
# =============>
batch_time = time.time() - start
summary_string = f'({i + 1}/{len(self.test_loader)}) | batch: {batch_time * 10.0:.4}ms | ' \
f'Total: {bar.elapsed_td} | ETA: {bar.eta_td:}'
bar.suffix = summary_string
bar.next()
bar.finish()
logger.info(summary_string)