def process(self, images, return_time=False): with torch.no_grad(): output = self.model(images)[-1] hm = output['hm'].sigmoid_() wh = output['wh'] reg = output['reg'] if self.opt.reg_offset else None if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() dets = ctdet_decode(hm, wh, reg=reg, K=self.opt.K, kernel=self.opt.nms_kernel) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): torch.cuda.synchronize() output = self.model(images)[-1] output['hm'] = output['hm'].sigmoid_() if self.opt.hm_hp and not self.opt.mse_loss: output['hm_hp'] = output['hm_hp'].sigmoid_() reg = output['reg'] if self.opt.reg_offset else None hm_hp = output['hm_hp'] if self.opt.hm_hp else None hp_offset = output['hp_offset'] if self.opt.reg_hp_offset else None torch.cuda.synchronize() forward_time = time.time() if self.opt.flip_test: output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2 output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2 output['hps'] = (output['hps'][0:1] + clothlandmark_flip_lr_off( output['hps'][1:2], self.flip_idx)) / 2 hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \ if hm_hp is not None else None reg = reg[0:1] if reg is not None else None hp_offset = hp_offset[0:1] if hp_offset is not None else None dets = deepfashion2_pose_decode(output['hm'], output['wh'], output['hps'], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): output = self.model(images)[-1] hm = output['hm'].sigmoid_() wh = output['wh'] #reg = output['reg'] if self.opt.reg_offset else None dep = 1. / (output['dep'].sigmoid() + 1e-6) - 1. if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() dets = ctdetplus_decode( dep, hm, wh, cat_spec_wh=self.opt.cat_spec_wh, K=self.opt.K) ##cambiado decode SE HA ELIMINADO REG if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): # feature map output = self.model(images)[-1] hm = output['hm'].sigmoid_() wh = output['wh'] reg = output['reg'] if self.opt.reg_offset else None if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() # dets shape: batchsize x K x 6 dets = ctdet_decode(hm, wh, reg=reg, cat_spec_wh=self.opt.cat_spec_wh, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): if False: output = self.model(images)[-1] hm = output['hm'].sigmoid_() wh = output['wh'] reg = output['reg'] if self.opt.reg_offset else None if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() dets = ctdet_decode(hm, wh, reg=reg, cat_spec_wh=self.opt.cat_spec_wh, K=self.opt.K) else: hm, wh, reg = self.model(images) torch.onnx.export(self.model, images, "ctdet-resdcn18.onnx", opset_version=9, verbose=False, output_names=["hm", "wh", "reg"]) quit() if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): output = self.model(images)[-1] # output sequence is [hm, wh, reg] hm = output[0].sigmoid_() wh = output[1] reg = output[2] if self.opt.reg_offset else None if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() dets = ctdet_decode(hm, wh, reg=reg, cat_spec_wh=self.opt.cat_spec_wh, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): torch.cuda.synchronize() outputs = self.model(images) hm, wh, hps, reg, hm_hp, hp_offset = outputs hm = hm.sigmoid_() if self.cfg.LOSS.HM_HP and not self.cfg.LOSS.MSE_LOSS: hm_hp = hm_hp.sigmoid_() reg = reg if self.cfg.LOSS.REG_OFFSET else None hm_hp = hm_hp if self.cfg.LOSS.HM_HP else None hp_offset = hp_offset if self.cfg.LOSS.REG_HP_OFFSET else None torch.cuda.synchronize() forward_time = time.time() if self.cfg.TEST.FLIP_TEST: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 hps = (hps[0:1] + flip_lr_off(hps[1:2], self.flip_idx)) / 2 hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \ if hm_hp is not None else None reg = reg[0:1] if reg is not None else None hp_offset = hp_offset[0:1] if hp_offset is not None else None dets = multi_pose_decode(hm, wh, hps, reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.cfg.TEST.TOPK) if return_time: return outputs, dets, forward_time else: return outputs, dets
def process(self, images, return_time=False): # output是各个head的字典,本模块对各个head处理,生成rbbox with torch.no_grad(): output = self.model(images)[-1] # import pdb; # pdb.set_trace() hm = output['hm'].sigmoid_() wh = output['wh'] angle = output['angle'] reg = output['reg'] if self.opt.reg_offset else None if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 # angle = (angle[0:1] + 2.0*np.pi-flip_tensor(angle[1:2])) / 2 angle = (angle[0:1] + 1.0 - flip_tensor(angle[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() dets = rodet_decode(hm, wh, angle, reg=reg, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): output = self.model(images)[-1] hm = output['hm'].sigmoid_() hp_offset = output['hp_offset'] hm_hp = output['hm_hp'] hps = output['hps'] reg = output['reg'] if self.opt.reg_offset else None if self.opt.flip_test: hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 reg = reg[0:1] if reg is not None else None torch.cuda.synchronize() forward_time = time.time() dets = ctdet_four_decode(hm, hps, reg=reg, K=self.opt.K) # dets = ctdet_four_decode(hm, hps, reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def multiscale_process(self, images, return_time=False): with torch.no_grad(): output = self.model(images)[-1] hm_small = output['hm_small'].sigmoid_() hm_medium = output['hm_medium'].sigmoid_() hm_big = output['hm_big'].sigmoid_() wh_small = output['wh_small'] wh_medium = output['wh_medium'] wh_big = output['wh_big'] reg_small = output['reg_small'] if self.opt.reg_offset else None reg_medium = output['reg_medium'] if self.opt.reg_offset else None reg_big = output['reg_big'] if self.opt.reg_offset else None scale = output['scale'] if self.opt.reg_proposal else None if self.opt.flip_test: hm_small = (hm_small[0:1] + flip_tensor(hm_small[1:2])) / 2 wh_small = (wh_small[0:1] + flip_tensor(wh_small[1:2])) / 2 reg_small = reg_small[0:1] if reg_small is not None else None hm_medium = (hm_medium[0:1] + flip_tensor(hm_medium[1:2])) / 2 wh_medium = (wh_medium[0:1] + flip_tensor(wh_medium[1:2])) / 2 reg_medium = reg_medium[0:1] if reg_medium is not None else None hm_big = (hm_big[0:1] + flip_tensor(hm_big[1:2])) / 2 wh_big = (wh_big[0:1] + flip_tensor(wh_big[1:2])) / 2 reg_big = reg_big[0:1] if reg_big is not None else None scale = (scale[0:1] + flip_tensor(scale[0:1])) / 2 torch.cuda.synchronize() forward_time = time.time() # dets = ctdet_decode(hm, wh, reg=reg, K=self.opt.K) dets = ctdet_multiscale_decode(hm_small, hm_medium, hm_big, wh_small, wh_medium, wh_big, reg_small, reg_medium, reg_big, scale=scale, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): output = self.model(images)[-1] if self.opt.mdn: BS = output['mdn_logits'].shape[0] M = self.opt.mdn_n_comps H, W = output['mdn_logits'].shape[-2:] K = self.opt.num_classes if self.opt.cat_spec_wh else 1 mdn_logits = output['mdn_logits'] mdn_logits = mdn_logits.reshape((BS, M, K, H, W)).permute( (2, 0, 1, 3, 4)) mdn_pi = torch.clamp( torch.nn.Softmax(dim=2)(mdn_logits), 1e-4, 1. - 1e-4) mdn_sigma = torch.clamp( torch.nn.ELU()(output['mdn_sigma']) + self.opt.mdn_min_sigma, 1e-4, 1e5) mdn_sigma = mdn_sigma.reshape((BS, M, 2, K, H, W)).permute( (3, 0, 1, 2, 4, 5)) #mdn_sigma = mdn_sigma.reshape((BS,M*2,K,H,W)).permute((2,0,1,3,4)) mdn_mu = output['wh'] mdn_mu = mdn_mu.reshape((BS, M, 2, K, H, W)).permute( (3, 0, 1, 2, 4, 5)) mdn_mu = mdn_mu.reshape((K * BS, M, 2, H, W)) mdn_sigma = mdn_sigma.reshape((K * BS, M, 2, H, W)) mdn_pi = mdn_pi.reshape((K * BS, M, H, W)) if self.opt.mdn_limit_comp is not None: cid = self.opt.mdn_limit_comp mdn_pi = mdn_pi[:, cid:cid + 1] mdn_sigma = mdn_sigma[:, cid:cid + 1] mdn_mu = mdn_mu[:, cid:cid + 1] M = 1 #print('mdn_mu.shape',mdn_mu.shape,'mdn_sigma.shape',mdn_sigma.shape,'mdn_pi.shape',mdn_pi.shape) C = 2 if self.opt.mdn_48: central = mdn_pi * torch.reciprocal( mdn_sigma[:, :, 0, :, :])**C * torch.reciprocal( mdn_sigma[:, :, 1, :, :])**C pi_max, pi_max_idx = torch.max(central, 1) else: pi_max, pi_max_idx = torch.max(mdn_pi, 1) if self.opt.mdn_max or self.opt.mdn_48: a = pi_max_idx.unsqueeze(1).repeat(1, C, 1, 1).reshape( BS * K, 1, C, H, W) wh = torch.gather(mdn_mu, 1, a).squeeze(1) a = pi_max_idx.unsqueeze(1).repeat(1, 2, 1, 1).reshape( BS * K, 1, 2, H, W) sigmas = torch.gather(mdn_sigma, 1, a).squeeze(1) elif self.opt.mdn_sum: wh = torch.sum(mdn_mu * mdn_pi.unsqueeze(2), 1) sigmas = torch.sum(mdn_sigma * mdn_pi.unsqueeze(2), 1) wh = wh.reshape((K, BS, 2, H, W)).permute( (1, 0, 2, 3, 4)).reshape((BS, 2 * K, H, W)) mdn_sigma = sigmas.reshape((K, BS, 2, H, W)).permute( (1, 0, 2, 3, 4)).reshape((BS, 2 * K, H, W)) mdn_pi = mdn_pi.reshape((K, BS, -1, H, W)).permute( (1, 0, 2, 3, 4)).reshape((BS, -1, H, W)) output.update({'wh': wh}) #if self.opt.debug == 4: output.update({ 'mdn_max_idx': pi_max_idx.unsqueeze(1), 'mdn_sigmas': mdn_sigma, 'mdn_max_pi': pi_max.unsqueeze(1) }) hm = output['hm'].sigmoid_() wh = output['wh'] # print('wh.shape',wh.shape,'hm.shape',hm.shape ) # wh.shape torch.Size([1, 160, <H>,<W>]) cswh # wh.shape torch.Size([1, 2, <H>,<W>]) reg = output['reg'] if self.opt.reg_offset else None if self.opt.flip_test: # if self.opts.mdn: # raise NotImplementedError hm = (hm[0:1] + flip_tensor(hm[1:2])) / 2 wh = (wh[0:1] + flip_tensor(wh[1:2])) / 2 reg = reg[0:1] if reg is not None else None if 'mdn_sigmas' in output: mdn_sigmas = output['mdn_sigmas'] output['mdn_sigmas'] = (mdn_sigmas[0:1] + flip_tensor(mdn_sigmas[1:2])) / 2 torch.cuda.synchronize() forward_time = time.time() dets = ctdet_decode(hm, wh, reg=reg, K=self.opt.K, cat_spec_wh=self.opt.cat_spec_wh, mdn_max_idx=output.get('mdn_max_idx'), mdn_max_pi=output.get('mdn_max_pi'), mdn_sigmas=output.get('mdn_sigmas')) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): torch.cuda.synchronize() if self.opt.zjb: all_out = self.model(images) output = all_out[-1] #gt output['hm_hp'] = self.gt[0]['hm_hp'].cuda() if self.opt.hm_hp and not self.opt.mse_loss: output['hm_hp'] = output['hm_hp'].sigmoid_() hm_hp = output['hm_hp'] if self.opt.hm_hp else None hp_offset = output[ 'hp_offset'] if self.opt.reg_hp_offset else None torch.cuda.synchronize() forward_time = time.time() #T-param # if self.opt.flip_test: # output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2 # output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2 # output['hps'] = (output['hps'][0:1] + # flip_lr_off(output['hps'][1:2], self.flip_idx)) / 2 # hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \ # if hm_hp is not None else None # reg = reg[0:1] if reg is not None else None # hp_offset = hp_offset[0:1] if hp_offset is not None else None dets, center = multi_pose_decode_c(output, output['hps'], hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K) dets = dets.data.cpu().numpy().reshape(2, -1, 8) center = center.data.cpu().numpy().reshape(2, -1, 4) dets = dets.reshape(1, -1, 8) center = center.reshape(1, -1, 4) detections = dets center_points = center classes = detections[..., -1] classes = classes[0] detections = detections[0] center_points = center_points[0] valid_ind = detections[:, 4] > -1 valid_detections = detections[valid_ind] box_width = valid_detections[:, 2] - valid_detections[:, 0] box_height = valid_detections[:, 3] - valid_detections[:, 1] s_ind = (box_width * box_height <= 22500) l_ind = (box_width * box_height > 22500) s_detections = valid_detections[s_ind] l_detections = valid_detections[l_ind] s_left_x = (2 * s_detections[:, 0] + s_detections[:, 2]) / 3 s_right_x = (s_detections[:, 0] + 2 * s_detections[:, 2]) / 3 s_top_y = (2 * s_detections[:, 1] + s_detections[:, 3]) / 3 s_bottom_y = (s_detections[:, 1] + 2 * s_detections[:, 3]) / 3 s_temp_score = copy.copy(s_detections[:, 4]) s_detections[:, 4] = -1 center_x = center_points[:, 0][:, np.newaxis] center_y = center_points[:, 1][:, np.newaxis] s_left_x = s_left_x[np.newaxis, :] s_right_x = s_right_x[np.newaxis, :] s_top_y = s_top_y[np.newaxis, :] s_bottom_y = s_bottom_y[np.newaxis, :] ind_lx = (center_x - s_left_x) > 0 ind_rx = (center_x - s_right_x) < 0 ind_ty = (center_y - s_top_y) > 0 ind_by = (center_y - s_bottom_y) < 0 ind_cls = (center_points[:, 2][:, np.newaxis] - s_detections[:, -1][np.newaxis, :]) == 0 ind_s_new_score = np.max( ((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) & (ind_by + 0) & (ind_cls + 0)), axis=0) == 1 index_s_new_score = np.argmax( ((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) & (ind_by + 0) & (ind_cls + 0))[:, ind_s_new_score], axis=0) s_detections[:, 4][ind_s_new_score] = ( s_temp_score[ind_s_new_score] * 2 + center_points[index_s_new_score, 3]) / 3 l_left_x = (3 * l_detections[:, 0] + 2 * l_detections[:, 2]) / 5 l_right_x = (2 * l_detections[:, 0] + 3 * l_detections[:, 2]) / 5 l_top_y = (3 * l_detections[:, 1] + 2 * l_detections[:, 3]) / 5 l_bottom_y = (2 * l_detections[:, 1] + 3 * l_detections[:, 3]) / 5 l_temp_score = copy.copy(l_detections[:, 4]) l_detections[:, 4] = -1 center_x = center_points[:, 0][:, np.newaxis] center_y = center_points[:, 1][:, np.newaxis] l_left_x = l_left_x[np.newaxis, :] l_right_x = l_right_x[np.newaxis, :] l_top_y = l_top_y[np.newaxis, :] l_bottom_y = l_bottom_y[np.newaxis, :] ind_lx = (center_x - l_left_x) > 0 ind_rx = (center_x - l_right_x) < 0 ind_ty = (center_y - l_top_y) > 0 ind_by = (center_y - l_bottom_y) < 0 ind_cls = (center_points[:, 2][:, np.newaxis] - l_detections[:, -1][np.newaxis, :]) == 0 ind_l_new_score = np.max( ((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) & (ind_by + 0) & (ind_cls + 0)), axis=0) == 1 index_l_new_score = np.argmax( ((ind_lx + 0) & (ind_rx + 0) & (ind_ty + 0) & (ind_by + 0) & (ind_cls + 0))[:, ind_l_new_score], axis=0) l_detections[:, 4][ind_l_new_score] = ( l_temp_score[ind_l_new_score] * 2 + center_points[index_l_new_score, 3]) / 3 detections = np.concatenate([l_detections, s_detections], axis=0) detections = detections[np.argsort(-detections[:, 4])] classes = detections[..., -1] #nms keep_inds = (detections[:, 4] > 0) #keep_inds = (detections[:, 4] > ) detections = detections[keep_inds] classes = classes[keep_inds] nms_inds = py_nms(detections[:, 0:5], 0.5) detections = detections[nms_inds] classes = classes[nms_inds] #gt # det_gt = self.gt[0]['gt_det'] # detections = np.ones((len(det_gt), 8)) # for i in range(len(det_gt)): # detections[i][:4] = det_gt[i].numpy() # detections[i][7] = 0 # # if detections.shape[0] == 0: # self.non_person += 1 # print("nonperson"+str(self.non_person)) # self.last_dets[0][0, :, 4] = 0 # self.last_dets[0] = np.zeros(shape=(1, 1, 6+34)) # if return_time: # return output, self.last_dets[0], forward_time # else: # return output, self.last_dets[0] kps = multi_pose_decode_c1(output, detections, output['hps'], hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K).data.cpu().numpy()[0] num_j = kps.shape[1] dets = np.zeros(shape=(1, detections.shape[0], 6 + num_j)) dets[0, :, 0:5] = detections[:, 0:5] dets[0, :, 5:5 + num_j] = kps dets[0, :, -1] = detections[:, -1] # top_bboxes[image_id] = {} # for j in range(categories): # keep_inds = (classes == j) # top_bboxes[image_id][j + 1] = detections[keep_inds][:, 0:7].astype(np.float32) # if merge_bbox: # soft_nms_merge(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm, weight_exp=weight_exp) # else: # soft_nms(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm) # top_bboxes[image_id][j + 1] = top_bboxes[image_id][j + 1][:, 0:5] # # scores = np.hstack([ # top_bboxes[image_id][j][:, -1] # for j in range(1, categories + 1) # ]) # if len(scores) > max_per_image: # kth = len(scores) - max_per_image # thresh = np.partition(scores, kth)[kth] # for j in range(1, categories + 1): # keep_inds = (top_bboxes[image_id][j][:, -1] >= thresh) # top_bboxes[image_id][j] = top_bboxes[image_id][j][keep_inds] self.last_dets[0] = dets if return_time: return output, dets, forward_time else: return output, dets else: output = self.model(images)[-1] output['hm'] = output['hm'].sigmoid_() if self.opt.hm_hp and not self.opt.mse_loss: output['hm_hp'] = output['hm_hp'].sigmoid_() reg = output['reg'] if self.opt.reg_offset else None hm_hp = output['hm_hp'] if self.opt.hm_hp else None hp_offset = output[ 'hp_offset'] if self.opt.reg_hp_offset else None torch.cuda.synchronize() forward_time = time.time() if self.opt.flip_test: output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2 output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2 output['hps'] = (output['hps'][0:1] + flip_lr_off( output['hps'][1:2], self.flip_idx)) / 2 hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \ if hm_hp is not None else None reg = reg[0:1] if reg is not None else None hp_offset = hp_offset[ 0:1] if hp_offset is not None else None dets = multi_pose_decode(output['hm'], output['wh'], output['hps'], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): torch.cuda.synchronize() output = self.model(images)[-1] # hm_x, hm_hp_x, wh, hps, reg, hp_offset, hm_max, hm_hp_max = self.model(images) # hm_x, hm_hp_x, wh, hps, reg, hp_offset = self.model(images) # # import planer # from planer import read_net # import cupy as cp # images_cp = cp.asarray(images.cpu().data.numpy()) # pal = planer.core(cp) # net = read_net('res18') # # hm_x, hm_hp_x, wh, hps, reg, hp_offset = net(images_cp) # hm_x = torch.tensor(cp.asnumpy(hm_x).astype('float32')).cuda() # hm_hp_x = torch.tensor(cp.asnumpy(hm_hp_x).astype('float32')).cuda() # wh = torch.tensor(cp.asnumpy(wh).astype('float32')).cuda() # hps = torch.tensor(cp.asnumpy(hps).astype('float32')).cuda() # reg = torch.tensor(cp.asnumpy(reg).astype('float32')).cuda() # hp_offset = torch.tensor(cp.asnumpy(hp_offset).astype('float32')).cuda() # output = {'hm': hm_x, 'hm_hp': hm_hp_x, 'wh': wh, 'hps': hps, 'reg': reg, # 'hp_offset': hp_offset, # # 'hm_max': hm_max, 'hm_hp_max': hm_hp_max # } output['hm'] = output['hm'].sigmoid_() # output['hm_max'] = output['hm_max'].sigmoid_() if self.opt.hm_hp and not self.opt.mse_loss: output['hm_hp'] = output['hm_hp'].sigmoid_() # output['hm_hp_max'] = output['hm_hp_max'].sigmoid_() reg = output['reg'] if self.opt.reg_offset else None hm_hp = output['hm_hp'] if self.opt.hm_hp else None hp_offset = output['hp_offset'] if self.opt.reg_hp_offset else None torch.cuda.synchronize() forward_time = time.time() if self.opt.flip_test: output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2 output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2 output['hps'] = (output['hps'][0:1] + flip_lr_off( output['hps'][1:2], self.flip_idx)) / 2 hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \ if hm_hp is not None else None reg = reg[0:1] if reg is not None else None hp_offset = hp_offset[0:1] if hp_offset is not None else None dets = multi_pose_decode( output['hm'], output['wh'], output['hps'], # output['hm_max'], output['hm_hp_max'], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K) if return_time: return output, dets, forward_time else: return output, dets
def process(self, images, return_time=False): with torch.no_grad(): torch.cuda.synchronize() output = self.model(images)[-1] if self.opt.mdn: mdn_logits = output['mdn_logits'] mdn_pi = torch.clamp(torch.nn.Softmax(dim=1)(mdn_logits), 1e-4, 1.-1e-4) mdn_sigma= torch.clamp(torch.nn.ELU()(output['mdn_sigma'])+self.opt.mdn_min_sigma,1e-4,1e5) mdn_mu = output['hps'] # print("mdn_pi.shape:",mdn_pi.shape) # print("mdn_mu.shape:",mdn_mu.shape) # print("mdn_sigma.shape:",mdn_sigma.shape) (BS,_,H,W) = mdn_sigma.shape if self.opt.mdn_limit_comp is not None: M= mdn_pi.shape[1] C = mdn_mu.shape[1]//M cid=self.opt.mdn_limit_comp mdn_pi = mdn_pi[:,cid:cid+1,:,:] mdn_sigma=mdn_sigma[:,2*cid:2*cid+2,:,:] mdn_mu = mdn_mu[:,C*cid:C*cid+C,:,:] M= mdn_pi.shape[1] mdn_sigma = torch.reshape(mdn_sigma, (BS,M,2,H,W)) C = mdn_mu.shape[1]//M mdn_mu = torch.reshape(mdn_mu, (BS,M,C,H,W)) if self.opt.mdn_48: central = mdn_pi * torch.reciprocal(mdn_sigma[:,:,0,:,:])**C * torch.reciprocal(mdn_sigma[:,:,1,:,:])**C pi_max,pi_max_idx = torch.max(central,1) else: pi_max,pi_max_idx = torch.max(mdn_pi,1) if self.opt.mdn_max or self.opt.mdn_48: a = pi_max_idx.unsqueeze(1).repeat(1,C,1,1).reshape(BS,1,C,H,W) hps = torch.gather(mdn_mu,1,a).squeeze(1) a = pi_max_idx.unsqueeze(1).repeat(1,2,1,1).reshape(BS,1,2,H,W) sigmas = torch.gather(mdn_sigma,1,a).squeeze(1) elif self.opt.mdn_sum: hps = torch.sum(mdn_mu*mdn_pi.unsqueeze(2),1) sigmas = torch.sum(mdn_sigma*mdn_pi.unsqueeze(2),1) output.update({'hps':hps}) #if self.opt.debug == 4: output.update({'mdn_max_idx':pi_max_idx.unsqueeze(1), 'mdn_sigmas':sigmas, 'mdn_max_pi':pi_max.unsqueeze(1) }) output['hm'] = output['hm'].sigmoid_() if self.opt.hm_hp: output['hm_hp'] = output['hm_hp'].sigmoid_() reg = output['reg'] if self.opt.reg_offset else None hm_hp = output['hm_hp'] if self.opt.hm_hp else None hp_offset = output['hp_offset'] if self.opt.reg_hp_offset else None torch.cuda.synchronize() forward_time = time.time() if self.opt.flip_test or self.opt.flip_test_max: output['hm'] = (output['hm'][0:1] + flip_tensor(output['hm'][1:2])) / 2 output['wh'] = (output['wh'][0:1] + flip_tensor(output['wh'][1:2])) / 2 hm_hp = (hm_hp[0:1] + flip_lr(hm_hp[1:2], self.flip_idx)) / 2 \ if hm_hp is not None else None reg = reg[0:1] if reg is not None else None hp_offset = hp_offset[0:1] if hp_offset is not None else None if self.opt.mdn: output['mdn_max_idx'][1:2] = flip_tensor(output['mdn_max_idx'][1:2]) output['mdn_max_pi'][1:2] = flip_tensor(output['mdn_max_pi'][1:2]) output['mdn_sigmas'][1:2] = flip_tensor(output['mdn_sigmas'][1:2]) if self.opt.flip_test: output['hps'] = (output['hps'][0:1] + flip_lr_off(output['hps'][1:2], self.flip_idx)) / 2 if self.opt.mdn: output['mdn_sigmas'] = (output['mdn_sigmas'][0:1] + output['mdn_sigmas'][1:2] ) / 2 elif self.opt.flip_test_max: if self.opt.mdn: output['hps'][1:2] = flip_lr_off(output['hps'][1:2], self.flip_idx) #print("output['mdn_max_pi'].shape:",output['mdn_max_pi'].shape) _,pi_max_idx = torch.max(output['mdn_max_pi'],0) _,_,H,W =output['hps'].shape a = pi_max_idx.unsqueeze(0).repeat(1,34,1,1).reshape(1,34,H,W) output['hps']= torch.gather(output['hps'],0,a) a = pi_max_idx.unsqueeze(0).repeat(1,2,1,1).reshape(1,2,H,W) output['mdn_sigmas']= torch.gather(output['mdn_sigmas'],0,a) dets = multi_pose_decode( output['hm'], output['wh'], output['hps'], reg=reg, hm_hp=hm_hp, hp_offset=hp_offset, K=self.opt.K, mdn_max_idx=output.get('mdn_max_idx'), mdn_max_pi=output.get('mdn_max_pi'), mdn_sigmas=output.get('mdn_sigmas')) if return_time: return output, dets, forward_time else: return output, dets