def vis_sequence(cfg,sequence, mv):
seq_data = parse_npz(sequence)
n_comps = seq_data['n_comps']
gender = seq_data['gender']
T = seq_data.n_frames
sbj_mesh = os.path.join(grab_path, '..', seq_data.body.vtemp)
sbj_vtemp = np.array(Mesh(filename=sbj_mesh).vertices)
sbj_m = smplx.create(model_path=cfg.model_path,
model_type='smplx',
gender=gender,
num_pca_comps=n_comps,
v_template=sbj_vtemp,
batch_size=T)
sbj_parms = params2torch(seq_data.body.params)
verts_sbj = to_cpu(sbj_m(**sbj_parms).vertices)
obj_mesh = os.path.join(grab_path, '..', seq_data.object.object_mesh)
obj_mesh = Mesh(filename=obj_mesh)
obj_vtemp = np.array(obj_mesh.vertices)
obj_m = ObjectModel(v_template=obj_vtemp,
batch_size=T)
obj_parms = params2torch(seq_data.object.params)
verts_obj = to_cpu(obj_m(**obj_parms).vertices)
table_mesh = os.path.join(grab_path, '..', seq_data.table.table_mesh)
table_mesh = Mesh(filename=table_mesh)
table_vtemp = np.array(table_mesh.vertices)
table_m = ObjectModel(v_template=table_vtemp,
batch_size=T)
table_parms = params2torch(seq_data.table.params)
verts_table = to_cpu(table_m(**table_parms).vertices)
skip_frame = 4
for frame in range(0,T, skip_frame):
o_mesh = Mesh(vertices=verts_obj[frame], faces=obj_mesh.faces, vc=colors['yellow'])
o_mesh.set_vertex_colors(vc=colors['red'], vertex_ids=seq_data['contact']['object'][frame] > 0)
s_mesh = Mesh(vertices=verts_sbj[frame], faces=sbj_m.faces, vc=colors['pink'], smooth=True)
s_mesh.set_vertex_colors(vc=colors['red'], vertex_ids=seq_data['contact']['body'][frame] > 0)
t_mesh = Mesh(vertices=verts_table[frame], faces=table_mesh.faces, vc=colors['white'])
mv.set_static_meshes([o_mesh, s_mesh, t_mesh])
def forward(self, x, targets=None):
img_dim = x.shape[2]
loss = 0
layer_outputs, yolo_outputs = [], []
for i, (module_def,
module) in enumerate(zip(self.module_defs, self.module_list)):
if module_def["type"] in ["convolutional", "upsample", "maxpool"]:
x = module(x)
elif module_def["type"] == "route":
x = torch.cat([
layer_outputs[int(layer_i)]
for layer_i in module_def["layers"].split(",")
], 1)
elif module_def["type"] == "shortcut":
layer_i = int(module_def["from"])
x = layer_outputs[-1] + layer_outputs[layer_i]
elif module_def["type"] == "yolo":
x, layer_loss = module[0](x, targets, img_dim)
loss += layer_loss
yolo_outputs.append(x)
layer_outputs.append(x)
yolo_outputs = to_cpu(torch.cat(yolo_outputs, 1))
return yolo_outputs if targets is None else (loss, yolo_outputs)
def save_grab_vertices(cfg, logger=None, **params):
grab_path = cfg.grab_path
out_path = cfg.out_path
makepath(out_path)
if logger is None:
logger = makelogger(log_dir=os.path.join(out_path,
'grab_preprocessing.log'),
mode='a').info
else:
logger = logger
logger('Starting to get vertices for GRAB!')
all_seqs = glob.glob(grab_path + '/*/*.npz')
logger('Total sequences: %d' % len(all_seqs))
# stime = datetime.now().replace(microsecond=0)
# shutil.copy2(sys.argv[0],
# os.path.join(out_path,
# os.path.basename(sys.argv[0]).replace('.py','_%s.py' % datetime.strftime(stime,'%Y%m%d_%H%M'))))
if out_path is None:
out_path = grab_path
for sequence in tqdm(all_seqs):
outfname = makepath(sequence.replace(grab_path, out_path).replace(
'.npz', '_verts_body.npz'),
isfile=True)
action_name = os.path.basename(sequence)
if os.path.exists(outfname):
logger('Results for %s split already exist.' % (action_name))
continue
else:
logger('Processing data for %s split.' % (action_name))
seq_data = parse_npz(sequence)
n_comps = seq_data['n_comps']
gender = seq_data['gender']
T = seq_data.n_frames
if cfg.save_body_verts:
sbj_mesh = os.path.join(grab_path, '..', seq_data.body.vtemp)
sbj_vtemp = np.array(Mesh(filename=sbj_mesh).vertices)
sbj_m = smplx.create(model_path=cfg.model_path,
model_type='smplx',
gender=gender,
num_pca_comps=n_comps,
v_template=sbj_vtemp,
batch_size=T)
sbj_parms = params2torch(seq_data.body.params)
verts_sbj = to_cpu(sbj_m(**sbj_parms).vertices)
np.savez_compressed(outfname, verts_body=verts_sbj)
if cfg.save_lhand_verts:
lh_mesh = os.path.join(grab_path, '..', seq_data.lhand.vtemp)
lh_vtemp = np.array(Mesh(filename=lh_mesh).vertices)
lh_m = smplx.create(model_path=cfg.model_path,
model_type='mano',
is_rhand=False,
v_template=lh_vtemp,
num_pca_comps=n_comps,
flat_hand_mean=True,
batch_size=T)
lh_parms = params2torch(seq_data.lhand.params)
verts_lh = to_cpu(lh_m(**lh_parms).vertices)
np.savez_compressed(outfname.replace('_verts_body.npz',
'_verts_lhand.npz'),
verts_body=verts_lh)
if cfg.save_rhand_verts:
rh_mesh = os.path.join(grab_path, '..', seq_data.rhand.vtemp)
rh_vtemp = np.array(Mesh(filename=rh_mesh).vertices)
rh_m = smplx.create(model_path=cfg.model_path,
model_type='mano',
is_rhand=True,
v_template=rh_vtemp,
num_pca_comps=n_comps,
flat_hand_mean=True,
batch_size=T)
rh_parms = params2torch(seq_data.body.params)
verts_rh = to_cpu(rh_m(**rh_parms).vertices)
np.savez_compressed(outfname.replace('_verts_body.npz',
'_verts_rhand.npz'),
verts_body=verts_rh)
if cfg.save_object_verts:
obj_mesh = os.path.join(grab_path, '..',
seq_data.object.object_mesh)
obj_vtemp = np.array(Mesh(filename=obj_mesh).vertices)
sample_id = np.random.choice(obj_vtemp.shape[0],
cfg.n_verts_sample,
replace=False)
obj_m = ObjectModel(v_template=obj_vtemp[sample_id], batch_size=T)
obj_parms = params2torch(seq_data.object.params)
verts_obj = to_cpu(obj_m(**obj_parms).vertices)
np.savez_compressed(outfname.replace('_verts_body.npz',
'_verts_object.npz'),
verts_object=verts_obj)
logger('Processing finished')
def forward(self, x, targets=None, img_dim=None):
# Tensors for cuda support
FloatTensor = torch.cuda.FloatTensor if x.is_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if x.is_cuda else torch.LongTensor
ByteTensor = torch.cuda.ByteTensor if x.is_cuda else torch.ByteTensor
self.img_dim = img_dim
num_samples = x.size(0)
grid_size = x.size(2)
prediction = (x.view(num_samples, self.num_anchors,
self.num_classes + 5, grid_size,
grid_size).permute(0, 1, 3, 4, 2).contiguous())
# Get outputs
x = torch.sigmoid(prediction[..., 0]) # Center x
y = torch.sigmoid(prediction[..., 1]) # Center y
w = prediction[..., 2] # Width
h = prediction[..., 3] # Height
pred_conf = torch.sigmoid(prediction[..., 4]) # Conf
pred_cls = torch.sigmoid(prediction[..., 5:]) # Cls pred.
# If grid size does not match current we compute new offsets
if grid_size != self.grid_size:
self.compute_grid_offsets(grid_size, cuda=x.is_cuda)
# Add offset and scale with anchors
pred_boxes = FloatTensor(prediction[..., :4].shape)
pred_boxes[..., 0] = x.data + self.grid_x
pred_boxes[..., 1] = y.data + self.grid_y
pred_boxes[..., 2] = torch.exp(w.data) * self.anchor_w
pred_boxes[..., 3] = torch.exp(h.data) * self.anchor_h
output = torch.cat(
(
pred_boxes.view(num_samples, -1, 4) * self.stride,
pred_conf.view(num_samples, -1, 1),
pred_cls.view(num_samples, -1, self.num_classes),
),
-1,
)
if targets is None:
return output, 0
else:
iou_scores, class_mask, obj_mask, noobj_mask, tx, ty, tw, th, tcls, tconf = build_targets(
pred_boxes=pred_boxes,
pred_cls=pred_cls,
target=targets,
anchors=self.scaled_anchors,
ignore_thres=self.ignore_thres,
)
obj_mask = obj_mask.bool() # convert int8 to bool
noobj_mask = noobj_mask.bool() # convert int8 to bool
# Loss : Mask outputs to ignore non-existing objects (except with conf. loss)
loss_x = self.mse_loss(x[obj_mask], tx[obj_mask])
loss_y = self.mse_loss(y[obj_mask], ty[obj_mask])
loss_w = self.mse_loss(w[obj_mask], tw[obj_mask])
loss_h = self.mse_loss(h[obj_mask], th[obj_mask])
loss_conf_obj = self.bce_loss(pred_conf[obj_mask], tconf[obj_mask])
loss_conf_noobj = self.bce_loss(pred_conf[noobj_mask],
tconf[noobj_mask])
loss_conf = self.obj_scale * loss_conf_obj + self.noobj_scale * loss_conf_noobj
loss_cls = self.bce_loss(pred_cls[obj_mask], tcls[obj_mask])
total_loss = loss_x + loss_y + loss_w + loss_h + loss_conf + loss_cls
# Metrics
cls_acc = 100 * class_mask[obj_mask].mean()
conf_obj = pred_conf[obj_mask].mean()
conf_noobj = pred_conf[noobj_mask].mean()
conf50 = (pred_conf > 0.5).float()
iou50 = (iou_scores > 0.5).float()
iou75 = (iou_scores > 0.75).float()
detected_mask = conf50 * class_mask * tconf
precision = torch.sum(
iou50 * detected_mask) / (conf50.sum() + 1e-16)
recall50 = torch.sum(
iou50 * detected_mask) / (obj_mask.sum() + 1e-16)
recall75 = torch.sum(
iou75 * detected_mask) / (obj_mask.sum() + 1e-16)
self.metrics = {
"loss": to_cpu(total_loss).item(),
"x": to_cpu(loss_x).item(),
"y": to_cpu(loss_y).item(),
"w": to_cpu(loss_w).item(),
"h": to_cpu(loss_h).item(),
"conf": to_cpu(loss_conf).item(),
"cls": to_cpu(loss_cls).item(),
"cls_acc": to_cpu(cls_acc).item(),
"recall50": to_cpu(recall50).item(),
"recall75": to_cpu(recall75).item(),
"precision": to_cpu(precision).item(),
"conf_obj": to_cpu(conf_obj).item(),
"conf_noobj": to_cpu(conf_noobj).item(),
"grid_size": grid_size,
}
return output, total_loss
def data_preprocessing(self,cfg):
# stime = datetime.now().replace(microsecond=0)
# shutil.copy2(sys.argv[0],
# os.path.join(self.out_path,
# os.path.basename(sys.argv[0]).replace('.py','_%s.py' % datetime.strftime(stime,'%Y%m%d_%H%M'))))
self.subject_mesh = {}
self.obj_info = {}
self.sbj_info = {}
for split in self.split_seqs.keys():
self.logger('Processing data for %s split.' % (split))
frame_names = []
body_data = {
'global_orient': [],'body_pose': [],'transl': [],
'right_hand_pose': [],'left_hand_pose': [],
'jaw_pose': [],'leye_pose': [],'reye_pose': [],
'expression': [],'fullpose': [],
'contact':[], 'verts' :[]
}
object_data ={'verts': [], 'global_orient': [], 'transl': [], 'contact': []}
lhand_data = {'verts': [], 'global_orient': [], 'hand_pose': [], 'transl': [], 'fullpose': []}
rhand_data = {'verts': [], 'global_orient': [], 'hand_pose': [], 'transl': [], 'fullpose': []}
for sequence in tqdm(self.split_seqs[split]):
seq_data = parse_npz(sequence)
obj_name = seq_data.obj_name
sbj_id = seq_data.sbj_id
n_comps = seq_data.n_comps
gender = seq_data.gender
frame_mask = self.filter_contact_frames(seq_data)
# total selectd frames
T = frame_mask.sum()
if T < 1:
continue # if no frame is selected continue to the next sequence
sbj_params = prepare_params(seq_data.body.params, frame_mask)
rh_params = prepare_params(seq_data.rhand.params, frame_mask)
lh_params = prepare_params(seq_data.lhand.params, frame_mask)
obj_params = prepare_params(seq_data.object.params, frame_mask)
append2dict(body_data, sbj_params)
append2dict(rhand_data, rh_params)
append2dict(lhand_data, lh_params)
append2dict(object_data, obj_params)
sbj_vtemp = self.load_sbj_verts(sbj_id, seq_data)
if cfg.save_body_verts:
sbj_m = smplx.create(model_path=cfg.model_path,
model_type='smplx',
gender=gender,
num_pca_comps=n_comps,
v_template=sbj_vtemp,
batch_size=T)
sbj_parms = params2torch(sbj_params)
verts_sbj = to_cpu(sbj_m(**sbj_parms).vertices)
body_data['verts'].append(verts_sbj)
if cfg.save_lhand_verts:
lh_mesh = os.path.join(grab_path, '..', seq_data.lhand.vtemp)
lh_vtemp = np.array(Mesh(filename=lh_mesh).vertices)
lh_m = smplx.create(model_path=cfg.model_path,
model_type='mano',
is_rhand=False,
v_template=lh_vtemp,
num_pca_comps=n_comps,
flat_hand_mean=True,
batch_size=T)
lh_parms = params2torch(lh_params)
verts_lh = to_cpu(lh_m(**lh_parms).vertices)
lhand_data['verts'].append(verts_lh)
if cfg.save_rhand_verts:
rh_mesh = os.path.join(grab_path, '..', seq_data.rhand.vtemp)
rh_vtemp = np.array(Mesh(filename=rh_mesh).vertices)
rh_m = smplx.create(model_path=cfg.model_path,
model_type='mano',
is_rhand=True,
v_template=rh_vtemp,
num_pca_comps=n_comps,
flat_hand_mean=True,
batch_size=T)
rh_parms = params2torch(rh_params)
verts_rh = to_cpu(rh_m(**rh_parms).vertices)
rhand_data['verts'].append(verts_rh)
### for objects
obj_info = self.load_obj_verts(obj_name, seq_data, cfg.n_verts_sample)
if cfg.save_object_verts:
obj_m = ObjectModel(v_template=obj_info['verts_sample'],
batch_size=T)
obj_parms = params2torch(obj_params)
verts_obj = to_cpu(obj_m(**obj_parms).vertices)
object_data['verts'].append(verts_obj)
if cfg.save_contact:
body_data['contact'].append(seq_data.contact.body[frame_mask])
object_data['contact'].append(seq_data.contact.object[frame_mask][:,obj_info['verts_sample_id']])
frame_names.extend(['%s_%s' % (sequence.split('.')[0], fId) for fId in np.arange(T)])
self.logger('Processing for %s split finished' % split)
self.logger('Total number of frames for %s split is:%d' % (split, len(frame_names)))
out_data = [body_data, rhand_data, lhand_data, object_data]
out_data_name = ['body_data', 'rhand_data', 'lhand_data', 'object_data']
for idx, data in enumerate(out_data):
data = np2torch(data)
data_name = out_data_name[idx]
outfname = makepath(os.path.join(self.out_path, split, '%s.pt' % data_name), isfile=True)
torch.save(data, outfname)
np.savez(os.path.join(self.out_path, split, 'frame_names.npz'), frame_names=frame_names)
np.save(os.path.join(self.out_path, 'obj_info.npy'), self.obj_info)
np.save(os.path.join(self.out_path, 'sbj_info.npy'), self.sbj_info)