def process(self, images, return_time=False):
with torch.no_grad():
output = self.model(images)[-1]
hm = output['hm'].sigmoid_()
wh = output['wh']
reg = output['reg'] if self.opt.reg_offset else None
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
dets = ctdet_decode(hm,
wh,
reg=reg,
K=self.opt.K,
kernel=self.opt.nms_kernel)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
torch.cuda.synchronize()
output = self.model(images)[-1]
output['hm'] = output['hm'].sigmoid_()
if self.opt.hm_hp and not self.opt.mse_loss:
output['hm_hp'] = output['hm_hp'].sigmoid_()
reg = output['reg'] if self.opt.reg_offset else None
hm_hp = output['hm_hp'] if self.opt.hm_hp else None
hp_offset = output['hp_offset'] if self.opt.reg_hp_offset else None
torch.cuda.synchronize()
forward_time = time.time()
if self.opt.flip_test:
output['hm'] = (output['hm'][0:1] +
flip_tensor(output['hm'][1:2])) / 2
output['wh'] = (output['wh'][0:1] +
flip_tensor(output['wh'][1:2])) / 2
output['hps'] = (output['hps'][0:1] +
clothlandmark_flip_lr_off(
output['hps'][1:2], self.flip_idx)) / 2
hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \
if hm_hp is not None else None
reg = reg[0:1] if reg is not None else None
hp_offset = hp_offset[0:1] if hp_offset is not None else None
dets = deepfashion2_pose_decode(output['hm'],
output['wh'],
output['hps'],
reg=reg,
hm_hp=hm_hp,
hp_offset=hp_offset,
K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
output = self.model(images)[-1]
hm = output['hm'].sigmoid_()
wh = output['wh']
#reg = output['reg'] if self.opt.reg_offset else None
dep = 1. / (output['dep'].sigmoid() + 1e-6) - 1.
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
dets = ctdetplus_decode(
dep, hm, wh, cat_spec_wh=self.opt.cat_spec_wh,
K=self.opt.K) ##cambiado decode SE HA ELIMINADO REG
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
# feature map
output = self.model(images)[-1]
hm = output['hm'].sigmoid_()
wh = output['wh']
reg = output['reg'] if self.opt.reg_offset else None
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
# dets shape: batchsize x K x 6
dets = ctdet_decode(hm, wh, reg=reg, cat_spec_wh=self.opt.cat_spec_wh, K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
if False:
output = self.model(images)[-1]
hm = output['hm'].sigmoid_()
wh = output['wh']
reg = output['reg'] if self.opt.reg_offset else None
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
dets = ctdet_decode(hm, wh, reg=reg, cat_spec_wh=self.opt.cat_spec_wh, K=self.opt.K)
else:
hm, wh, reg = self.model(images)
torch.onnx.export(self.model, images, "ctdet-resdcn18.onnx", opset_version=9, verbose=False, output_names=["hm", "wh", "reg"])
quit()
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
output = self.model(images)[-1]
# output sequence is [hm, wh, reg]
hm = output[0].sigmoid_()
wh = output[1]
reg = output[2] if self.opt.reg_offset else None
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
dets = ctdet_decode(hm,
wh,
reg=reg,
cat_spec_wh=self.opt.cat_spec_wh,
K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
torch.cuda.synchronize()
outputs = self.model(images)
hm, wh, hps, reg, hm_hp, hp_offset = outputs
hm = hm.sigmoid_()
if self.cfg.LOSS.HM_HP and not self.cfg.LOSS.MSE_LOSS:
hm_hp = hm_hp.sigmoid_()
reg = reg if self.cfg.LOSS.REG_OFFSET else None
hm_hp = hm_hp if self.cfg.LOSS.HM_HP else None
hp_offset = hp_offset if self.cfg.LOSS.REG_HP_OFFSET else None
torch.cuda.synchronize()
forward_time = time.time()
if self.cfg.TEST.FLIP_TEST:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
hps = (hps[0:1] + flip_lr_off(hps[1:2], self.flip_idx)) / 2
hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \
if hm_hp is not None else None
reg = reg[0:1] if reg is not None else None
hp_offset = hp_offset[0:1] if hp_offset is not None else None
dets = multi_pose_decode(hm,
wh,
hps,
reg=reg,
hm_hp=hm_hp,
hp_offset=hp_offset,
K=self.cfg.TEST.TOPK)
if return_time:
return outputs, dets, forward_time
else:
return outputs, dets
def process(self, images, return_time=False):
# output是各个head的字典，本模块对各个head处理，生成rbbox
with torch.no_grad():
output = self.model(images)[-1]
# import pdb;
# pdb.set_trace()
hm = output['hm'].sigmoid_()
wh = output['wh']
angle = output['angle']
reg = output['reg'] if self.opt.reg_offset else None
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
# angle = (angle[0:1] + 2.0*np.pi-flip_tensor(angle[1:2])) / 2
angle = (angle[0:1] + 1.0 - flip_tensor(angle[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
dets = rodet_decode(hm, wh, angle, reg=reg, K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
output = self.model(images)[-1]
hm = output['hm'].sigmoid_()
hp_offset = output['hp_offset']
hm_hp = output['hm_hp']
hps = output['hps']
reg = output['reg'] if self.opt.reg_offset else None
if self.opt.flip_test:
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
reg = reg[0:1] if reg is not None else None
torch.cuda.synchronize()
forward_time = time.time()
dets = ctdet_four_decode(hm, hps, reg=reg, K=self.opt.K)
# dets = ctdet_four_decode(hm, hps, reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def multiscale_process(self, images, return_time=False):
with torch.no_grad():
output = self.model(images)[-1]
hm_small = output['hm_small'].sigmoid_()
hm_medium = output['hm_medium'].sigmoid_()
hm_big = output['hm_big'].sigmoid_()
wh_small = output['wh_small']
wh_medium = output['wh_medium']
wh_big = output['wh_big']
reg_small = output['reg_small'] if self.opt.reg_offset else None
reg_medium = output['reg_medium'] if self.opt.reg_offset else None
reg_big = output['reg_big'] if self.opt.reg_offset else None
scale = output['scale'] if self.opt.reg_proposal else None
if self.opt.flip_test:
hm_small = (hm_small[0:1] + flip_tensor(hm_small[1:2])) / 2
wh_small = (wh_small[0:1] + flip_tensor(wh_small[1:2])) / 2
reg_small = reg_small[0:1] if reg_small is not None else None
hm_medium = (hm_medium[0:1] + flip_tensor(hm_medium[1:2])) / 2
wh_medium = (wh_medium[0:1] + flip_tensor(wh_medium[1:2])) / 2
reg_medium = reg_medium[0:1] if reg_medium is not None else None
hm_big = (hm_big[0:1] + flip_tensor(hm_big[1:2])) / 2
wh_big = (wh_big[0:1] + flip_tensor(wh_big[1:2])) / 2
reg_big = reg_big[0:1] if reg_big is not None else None
scale = (scale[0:1] + flip_tensor(scale[0:1])) / 2
torch.cuda.synchronize()
forward_time = time.time()
# dets = ctdet_decode(hm, wh, reg=reg, K=self.opt.K)
dets = ctdet_multiscale_decode(hm_small, hm_medium, hm_big,
wh_small, wh_medium, wh_big,
reg_small, reg_medium, reg_big,
scale=scale, K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
output = self.model(images)[-1]
if self.opt.mdn:
BS = output['mdn_logits'].shape[0]
M = self.opt.mdn_n_comps
H, W = output['mdn_logits'].shape[-2:]
K = self.opt.num_classes if self.opt.cat_spec_wh else 1
mdn_logits = output['mdn_logits']
mdn_logits = mdn_logits.reshape((BS, M, K, H, W)).permute(
(2, 0, 1, 3, 4))
mdn_pi = torch.clamp(
torch.nn.Softmax(dim=2)(mdn_logits), 1e-4, 1. - 1e-4)
mdn_sigma = torch.clamp(
torch.nn.ELU()(output['mdn_sigma']) +
self.opt.mdn_min_sigma, 1e-4, 1e5)
mdn_sigma = mdn_sigma.reshape((BS, M, 2, K, H, W)).permute(
(3, 0, 1, 2, 4, 5))
#mdn_sigma = mdn_sigma.reshape((BS,M*2,K,H,W)).permute((2,0,1,3,4))
mdn_mu = output['wh']
mdn_mu = mdn_mu.reshape((BS, M, 2, K, H, W)).permute(
(3, 0, 1, 2, 4, 5))
mdn_mu = mdn_mu.reshape((K * BS, M, 2, H, W))
mdn_sigma = mdn_sigma.reshape((K * BS, M, 2, H, W))
mdn_pi = mdn_pi.reshape((K * BS, M, H, W))
if self.opt.mdn_limit_comp is not None:
cid = self.opt.mdn_limit_comp
mdn_pi = mdn_pi[:, cid:cid + 1]
mdn_sigma = mdn_sigma[:, cid:cid + 1]
mdn_mu = mdn_mu[:, cid:cid + 1]
M = 1
#print('mdn_mu.shape',mdn_mu.shape,'mdn_sigma.shape',mdn_sigma.shape,'mdn_pi.shape',mdn_pi.shape)
C = 2
if self.opt.mdn_48:
central = mdn_pi * torch.reciprocal(
mdn_sigma[:, :, 0, :, :])**C * torch.reciprocal(
mdn_sigma[:, :, 1, :, :])**C
pi_max, pi_max_idx = torch.max(central, 1)
else:
pi_max, pi_max_idx = torch.max(mdn_pi, 1)
if self.opt.mdn_max or self.opt.mdn_48:
a = pi_max_idx.unsqueeze(1).repeat(1, C, 1, 1).reshape(
BS * K, 1, C, H, W)
wh = torch.gather(mdn_mu, 1, a).squeeze(1)
a = pi_max_idx.unsqueeze(1).repeat(1, 2, 1, 1).reshape(
BS * K, 1, 2, H, W)
sigmas = torch.gather(mdn_sigma, 1, a).squeeze(1)
elif self.opt.mdn_sum:
wh = torch.sum(mdn_mu * mdn_pi.unsqueeze(2), 1)
sigmas = torch.sum(mdn_sigma * mdn_pi.unsqueeze(2), 1)
wh = wh.reshape((K, BS, 2, H, W)).permute(
(1, 0, 2, 3, 4)).reshape((BS, 2 * K, H, W))
mdn_sigma = sigmas.reshape((K, BS, 2, H, W)).permute(
(1, 0, 2, 3, 4)).reshape((BS, 2 * K, H, W))
mdn_pi = mdn_pi.reshape((K, BS, -1, H, W)).permute(
(1, 0, 2, 3, 4)).reshape((BS, -1, H, W))
output.update({'wh': wh})
#if self.opt.debug == 4:
output.update({
'mdn_max_idx': pi_max_idx.unsqueeze(1),
'mdn_sigmas': mdn_sigma,
'mdn_max_pi': pi_max.unsqueeze(1)
})
hm = output['hm'].sigmoid_()
wh = output['wh']
# print('wh.shape',wh.shape,'hm.shape',hm.shape )
# wh.shape torch.Size([1, 160, <H>,<W>]) cswh
# wh.shape torch.Size([1, 2, <H>,<W>])
reg = output['reg'] if self.opt.reg_offset else None
if self.opt.flip_test:
# if self.opts.mdn:
# raise NotImplementedError
hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2
wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2
reg = reg[0:1] if reg is not None else None
if 'mdn_sigmas' in output:
mdn_sigmas = output['mdn_sigmas']
output['mdn_sigmas'] = (mdn_sigmas[0:1] +
flip_tensor(mdn_sigmas[1:2])) / 2
torch.cuda.synchronize()
forward_time = time.time()
dets = ctdet_decode(hm,
wh,
reg=reg,
K=self.opt.K,
cat_spec_wh=self.opt.cat_spec_wh,
mdn_max_idx=output.get('mdn_max_idx'),
mdn_max_pi=output.get('mdn_max_pi'),
mdn_sigmas=output.get('mdn_sigmas'))
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
torch.cuda.synchronize()
if self.opt.zjb:
all_out = self.model(images)
output = all_out[-1]
#gt
output['hm_hp'] = self.gt[0]['hm_hp'].cuda()
if self.opt.hm_hp and not self.opt.mse_loss:
output['hm_hp'] = output['hm_hp'].sigmoid_()
hm_hp = output['hm_hp'] if self.opt.hm_hp else None
hp_offset = output[
'hp_offset'] if self.opt.reg_hp_offset else None
torch.cuda.synchronize()
forward_time = time.time()
#T-param
# if self.opt.flip_test:
# output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2
# output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2
# output['hps'] = (output['hps'][0:1] +
# flip_lr_off(output['hps'][1:2], self.flip_idx)) / 2
# hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \
# if hm_hp is not None else None
# reg = reg[0:1] if reg is not None else None
# hp_offset = hp_offset[0:1] if hp_offset is not None else None
dets, center = multi_pose_decode_c(output,
output['hps'],
hm_hp=hm_hp,
hp_offset=hp_offset,
K=self.opt.K)
dets = dets.data.cpu().numpy().reshape(2, -1, 8)
center = center.data.cpu().numpy().reshape(2, -1, 4)
dets = dets.reshape(1, -1, 8)
center = center.reshape(1, -1, 4)
detections = dets
center_points = center
classes = detections[..., -1]
classes = classes[0]
detections = detections[0]
center_points = center_points[0]
valid_ind = detections[:, 4] > -1
valid_detections = detections[valid_ind]
box_width = valid_detections[:, 2] - valid_detections[:, 0]
box_height = valid_detections[:, 3] - valid_detections[:, 1]
s_ind = (box_width * box_height <= 22500)
l_ind = (box_width * box_height > 22500)
s_detections = valid_detections[s_ind]
l_detections = valid_detections[l_ind]
s_left_x = (2 * s_detections[:, 0] + s_detections[:, 2]) / 3
s_right_x = (s_detections[:, 0] + 2 * s_detections[:, 2]) / 3
s_top_y = (2 * s_detections[:, 1] + s_detections[:, 3]) / 3
s_bottom_y = (s_detections[:, 1] + 2 * s_detections[:, 3]) / 3
s_temp_score = copy.copy(s_detections[:, 4])
s_detections[:, 4] = -1
center_x = center_points[:, 0][:, np.newaxis]
center_y = center_points[:, 1][:, np.newaxis]
s_left_x = s_left_x[np.newaxis, :]
s_right_x = s_right_x[np.newaxis, :]
s_top_y = s_top_y[np.newaxis, :]
s_bottom_y = s_bottom_y[np.newaxis, :]
ind_lx = (center_x - s_left_x) > 0
ind_rx = (center_x - s_right_x) < 0
ind_ty = (center_y - s_top_y) > 0
ind_by = (center_y - s_bottom_y) < 0
ind_cls = (center_points[:, 2][:, np.newaxis] -
s_detections[:, -1][np.newaxis, :]) == 0
ind_s_new_score = np.max(
((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) &
(ind_by + 0) & (ind_cls + 0)),
axis=0) == 1
index_s_new_score = np.argmax(
((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) & (ind_by + 0)
& (ind_cls + 0))[:, ind_s_new_score],
axis=0)
s_detections[:, 4][ind_s_new_score] = (
s_temp_score[ind_s_new_score] * 2 +
center_points[index_s_new_score, 3]) / 3
l_left_x = (3 * l_detections[:, 0] +
2 * l_detections[:, 2]) / 5
l_right_x = (2 * l_detections[:, 0] +
3 * l_detections[:, 2]) / 5
l_top_y = (3 * l_detections[:, 1] + 2 * l_detections[:, 3]) / 5
l_bottom_y = (2 * l_detections[:, 1] +
3 * l_detections[:, 3]) / 5
l_temp_score = copy.copy(l_detections[:, 4])
l_detections[:, 4] = -1
center_x = center_points[:, 0][:, np.newaxis]
center_y = center_points[:, 1][:, np.newaxis]
l_left_x = l_left_x[np.newaxis, :]
l_right_x = l_right_x[np.newaxis, :]
l_top_y = l_top_y[np.newaxis, :]
l_bottom_y = l_bottom_y[np.newaxis, :]
ind_lx = (center_x - l_left_x) > 0
ind_rx = (center_x - l_right_x) < 0
ind_ty = (center_y - l_top_y) > 0
ind_by = (center_y - l_bottom_y) < 0
ind_cls = (center_points[:, 2][:, np.newaxis] -
l_detections[:, -1][np.newaxis, :]) == 0
ind_l_new_score = np.max(
((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) &
(ind_by + 0) & (ind_cls + 0)),
axis=0) == 1
index_l_new_score = np.argmax(
((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) & (ind_by + 0)
& (ind_cls + 0))[:, ind_l_new_score],
axis=0)
l_detections[:, 4][ind_l_new_score] = (
l_temp_score[ind_l_new_score] * 2 +
center_points[index_l_new_score, 3]) / 3
detections = np.concatenate([l_detections, s_detections],
axis=0)
detections = detections[np.argsort(-detections[:, 4])]
classes = detections[..., -1]
#nms
keep_inds = (detections[:, 4] > 0)
#keep_inds = (detections[:, 4] > )
detections = detections[keep_inds]
classes = classes[keep_inds]
nms_inds = py_nms(detections[:, 0:5], 0.5)
detections = detections[nms_inds]
classes = classes[nms_inds]
#gt
# det_gt = self.gt[0]['gt_det']
# detections = np.ones((len(det_gt), 8))
# for i in range(len(det_gt)):
# detections[i][:4] = det_gt[i].numpy()
# detections[i][7] = 0
#
# if detections.shape[0] == 0:
# self.non_person += 1
# print("nonperson"+str(self.non_person))
# self.last_dets[0][0, :, 4] = 0
# self.last_dets[0] = np.zeros(shape=(1, 1, 6+34))
# if return_time:
# return output, self.last_dets[0], forward_time
# else:
# return output, self.last_dets[0]
kps = multi_pose_decode_c1(output,
detections,
output['hps'],
hm_hp=hm_hp,
hp_offset=hp_offset,
K=self.opt.K).data.cpu().numpy()[0]
num_j = kps.shape[1]
dets = np.zeros(shape=(1, detections.shape[0], 6 + num_j))
dets[0, :, 0:5] = detections[:, 0:5]
dets[0, :, 5:5 + num_j] = kps
dets[0, :, -1] = detections[:, -1]
# top_bboxes[image_id] = {}
# for j in range(categories):
# keep_inds = (classes == j)
# top_bboxes[image_id][j + 1] = detections[keep_inds][:, 0:7].astype(np.float32)
# if merge_bbox:
# soft_nms_merge(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm, weight_exp=weight_exp)
# else:
# soft_nms(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm)
# top_bboxes[image_id][j + 1] = top_bboxes[image_id][j + 1][:, 0:5]
#
# scores = np.hstack([
# top_bboxes[image_id][j][:, -1]
# for j in range(1, categories + 1)
# ])
# if len(scores) > max_per_image:
# kth = len(scores) - max_per_image
# thresh = np.partition(scores, kth)[kth]
# for j in range(1, categories + 1):
# keep_inds = (top_bboxes[image_id][j][:, -1] >= thresh)
# top_bboxes[image_id][j] = top_bboxes[image_id][j][keep_inds]
self.last_dets[0] = dets
if return_time:
return output, dets, forward_time
else:
return output, dets
else:
output = self.model(images)[-1]
output['hm'] = output['hm'].sigmoid_()
if self.opt.hm_hp and not self.opt.mse_loss:
output['hm_hp'] = output['hm_hp'].sigmoid_()
reg = output['reg'] if self.opt.reg_offset else None
hm_hp = output['hm_hp'] if self.opt.hm_hp else None
hp_offset = output[
'hp_offset'] if self.opt.reg_hp_offset else None
torch.cuda.synchronize()
forward_time = time.time()
if self.opt.flip_test:
output['hm'] = (output['hm'][0:1] +
flip_tensor(output['hm'][1:2])) / 2
output['wh'] = (output['wh'][0:1] +
flip_tensor(output['wh'][1:2])) / 2
output['hps'] = (output['hps'][0:1] + flip_lr_off(
output['hps'][1:2], self.flip_idx)) / 2
hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \
if hm_hp is not None else None
reg = reg[0:1] if reg is not None else None
hp_offset = hp_offset[
0:1] if hp_offset is not None else None
dets = multi_pose_decode(output['hm'],
output['wh'],
output['hps'],
reg=reg,
hm_hp=hm_hp,
hp_offset=hp_offset,
K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
torch.cuda.synchronize()
output = self.model(images)[-1]
# hm_x, hm_hp_x, wh, hps, reg, hp_offset, hm_max, hm_hp_max = self.model(images)
# hm_x, hm_hp_x, wh, hps, reg, hp_offset = self.model(images)
#
# import planer
# from planer import read_net
# import cupy as cp
# images_cp = cp.asarray(images.cpu().data.numpy())
# pal = planer.core(cp)
# net = read_net('res18')
#
# hm_x, hm_hp_x, wh, hps, reg, hp_offset = net(images_cp)
# hm_x = torch.tensor(cp.asnumpy(hm_x).astype('float32')).cuda()
# hm_hp_x = torch.tensor(cp.asnumpy(hm_hp_x).astype('float32')).cuda()
# wh = torch.tensor(cp.asnumpy(wh).astype('float32')).cuda()
# hps = torch.tensor(cp.asnumpy(hps).astype('float32')).cuda()
# reg = torch.tensor(cp.asnumpy(reg).astype('float32')).cuda()
# hp_offset = torch.tensor(cp.asnumpy(hp_offset).astype('float32')).cuda()
# output = {'hm': hm_x, 'hm_hp': hm_hp_x, 'wh': wh, 'hps': hps, 'reg': reg,
# 'hp_offset': hp_offset,
# # 'hm_max': hm_max, 'hm_hp_max': hm_hp_max
# }
output['hm'] = output['hm'].sigmoid_()
# output['hm_max'] = output['hm_max'].sigmoid_()
if self.opt.hm_hp and not self.opt.mse_loss:
output['hm_hp'] = output['hm_hp'].sigmoid_()
# output['hm_hp_max'] = output['hm_hp_max'].sigmoid_()
reg = output['reg'] if self.opt.reg_offset else None
hm_hp = output['hm_hp'] if self.opt.hm_hp else None
hp_offset = output['hp_offset'] if self.opt.reg_hp_offset else None
torch.cuda.synchronize()
forward_time = time.time()
if self.opt.flip_test:
output['hm'] = (output['hm'][0:1] +
flip_tensor(output['hm'][1:2])) / 2
output['wh'] = (output['wh'][0:1] +
flip_tensor(output['wh'][1:2])) / 2
output['hps'] = (output['hps'][0:1] + flip_lr_off(
output['hps'][1:2], self.flip_idx)) / 2
hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \
if hm_hp is not None else None
reg = reg[0:1] if reg is not None else None
hp_offset = hp_offset[0:1] if hp_offset is not None else None
dets = multi_pose_decode(
output['hm'],
output['wh'],
output['hps'],
# output['hm_max'], output['hm_hp_max'],
reg=reg,
hm_hp=hm_hp,
hp_offset=hp_offset,
K=self.opt.K)
if return_time:
return output, dets, forward_time
else:
return output, dets
def process(self, images, return_time=False):
with torch.no_grad():
torch.cuda.synchronize()
output = self.model(images)[-1]
if self.opt.mdn:
mdn_logits = output['mdn_logits']
mdn_pi = torch.clamp(torch.nn.Softmax(dim=1)(mdn_logits), 1e-4, 1.-1e-4)
mdn_sigma= torch.clamp(torch.nn.ELU()(output['mdn_sigma'])+self.opt.mdn_min_sigma,1e-4,1e5)
mdn_mu = output['hps']
# print("mdn_pi.shape:",mdn_pi.shape)
# print("mdn_mu.shape:",mdn_mu.shape)
# print("mdn_sigma.shape:",mdn_sigma.shape)
(BS,_,H,W) = mdn_sigma.shape
if self.opt.mdn_limit_comp is not None:
M= mdn_pi.shape[1]
C = mdn_mu.shape[1]//M
cid=self.opt.mdn_limit_comp
mdn_pi = mdn_pi[:,cid:cid+1,:,:]
mdn_sigma=mdn_sigma[:,2*cid:2*cid+2,:,:]
mdn_mu = mdn_mu[:,C*cid:C*cid+C,:,:]
M= mdn_pi.shape[1]
mdn_sigma = torch.reshape(mdn_sigma, (BS,M,2,H,W))
C = mdn_mu.shape[1]//M
mdn_mu = torch.reshape(mdn_mu, (BS,M,C,H,W))
if self.opt.mdn_48:
central = mdn_pi * torch.reciprocal(mdn_sigma[:,:,0,:,:])**C * torch.reciprocal(mdn_sigma[:,:,1,:,:])**C
pi_max,pi_max_idx = torch.max(central,1)
else:
pi_max,pi_max_idx = torch.max(mdn_pi,1)
if self.opt.mdn_max or self.opt.mdn_48:
a = pi_max_idx.unsqueeze(1).repeat(1,C,1,1).reshape(BS,1,C,H,W)
hps = torch.gather(mdn_mu,1,a).squeeze(1)
a = pi_max_idx.unsqueeze(1).repeat(1,2,1,1).reshape(BS,1,2,H,W)
sigmas = torch.gather(mdn_sigma,1,a).squeeze(1)
elif self.opt.mdn_sum:
hps = torch.sum(mdn_mu*mdn_pi.unsqueeze(2),1)
sigmas = torch.sum(mdn_sigma*mdn_pi.unsqueeze(2),1)
output.update({'hps':hps})
#if self.opt.debug == 4:
output.update({'mdn_max_idx':pi_max_idx.unsqueeze(1),
'mdn_sigmas':sigmas,
'mdn_max_pi':pi_max.unsqueeze(1)
})
output['hm'] = output['hm'].sigmoid_()
if self.opt.hm_hp:
output['hm_hp'] = output['hm_hp'].sigmoid_()
reg = output['reg'] if self.opt.reg_offset else None
hm_hp = output['hm_hp'] if self.opt.hm_hp else None
hp_offset = output['hp_offset'] if self.opt.reg_hp_offset else None
torch.cuda.synchronize()
forward_time = time.time()
if self.opt.flip_test or self.opt.flip_test_max:
output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2
output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2
hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \
if hm_hp is not None else None
reg = reg[0:1] if reg is not None else None
hp_offset = hp_offset[0:1] if hp_offset is not None else None
if self.opt.mdn:
output['mdn_max_idx'][1:2] = flip_tensor(output['mdn_max_idx'][1:2])
output['mdn_max_pi'][1:2] = flip_tensor(output['mdn_max_pi'][1:2])
output['mdn_sigmas'][1:2] = flip_tensor(output['mdn_sigmas'][1:2])
if self.opt.flip_test:
output['hps'] = (output['hps'][0:1] +
flip_lr_off(output['hps'][1:2], self.flip_idx)) / 2
if self.opt.mdn:
output['mdn_sigmas'] = (output['mdn_sigmas'][0:1] + output['mdn_sigmas'][1:2] ) / 2
elif self.opt.flip_test_max:
if self.opt.mdn:
output['hps'][1:2] = flip_lr_off(output['hps'][1:2], self.flip_idx)
#print("output['mdn_max_pi'].shape:",output['mdn_max_pi'].shape)
_,pi_max_idx = torch.max(output['mdn_max_pi'],0)
_,_,H,W =output['hps'].shape
a = pi_max_idx.unsqueeze(0).repeat(1,34,1,1).reshape(1,34,H,W)
output['hps']= torch.gather(output['hps'],0,a)
a = pi_max_idx.unsqueeze(0).repeat(1,2,1,1).reshape(1,2,H,W)
output['mdn_sigmas']= torch.gather(output['mdn_sigmas'],0,a)
dets = multi_pose_decode(
output['hm'], output['wh'], output['hps'],
reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K,
mdn_max_idx=output.get('mdn_max_idx'),
mdn_max_pi=output.get('mdn_max_pi'),
mdn_sigmas=output.get('mdn_sigmas'))
if return_time:
return output, dets, forward_time
else:
return output, dets
