def debug(self, batch, output, iter_id):
opt = self.opt
wh = output['wh'] if opt.reg_bbox else None
reg = output['reg'] if opt.reg_offset else None
dets = ddd_decode(output['hm'], output['rot'], output['dep'],
output['dim'], wh=wh, reg=reg, K=opt.K)
# x, y, score, r1-r8, depth, dim1-dim3, cls
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
calib = batch['meta']['calib'].detach().numpy()
# x, y, score, rot, depth, dim1, dim2, dim3
# if opt.dataset == 'gta':
# dets[:, 12:15] /= 3
dets_pred = ddd_post_process(
dets.copy(), batch['meta']['c'].detach().numpy(),
batch['meta']['s'].detach().numpy(), calib, opt)
dets_gt = ddd_post_process(
batch['meta']['gt_det'].detach().numpy().copy(),
batch['meta']['c'].detach().numpy(),
batch['meta']['s'].detach().numpy(), calib, opt)
# for i in range(input.size(0)):
for i in range(1):
debugger = Debugger(dataset=opt.dataset, ipynb=(opt.debug == 3),
theme=opt.debugger_theme)
img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = ((img * self.opt.std + self.opt.mean) * 255.).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'hm_pred')
debugger.add_blend_img(img, gt, 'hm_gt')
# decode
debugger.add_ct_detection(
img, dets[i], show_box=opt.reg_bbox, center_thresh=opt.center_thresh,
img_id='det_pred')
debugger.add_ct_detection(
img, batch['meta']['gt_det'][i].cpu().numpy().copy(),
show_box=opt.reg_bbox, img_id='det_gt')
debugger.add_3d_detection(
batch['meta']['image_path'][i], dets_pred[i], calib[i],
center_thresh=opt.center_thresh, img_id='add_pred')
debugger.add_3d_detection(
batch['meta']['image_path'][i], dets_gt[i], calib[i],
center_thresh=opt.center_thresh, img_id='add_gt')
# debugger.add_bird_view(
# dets_pred[i], center_thresh=opt.center_thresh, img_id='bird_pred')
# debugger.add_bird_view(dets_gt[i], img_id='bird_gt')
debugger.add_bird_views(
dets_pred[i], dets_gt[i],
center_thresh=opt.center_thresh, img_id='bird_pred_gt')
# debugger.add_blend_img(img, pred, 'out', white=True)
debugger.compose_vis_add(
batch['meta']['image_path'][i], dets_pred[i], calib[i],
opt.center_thresh, pred, 'bird_pred_gt', img_id='out')
# debugger.add_img(img, img_id='out')
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir, prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def debug(self, batch, output, iter_id):
opt = self.opt
reg = output['reg'] if opt.reg_offset else None
dets = ctdet_decode(output['hm'],
output['wh'],
reg=reg,
cat_spec_wh=opt.cat_spec_wh,
K=opt.K)
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
dets[:, :, :4] *= opt.down_ratio
dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2])
dets_gt[:, :, :4] *= opt.down_ratio
for i in range(1):
debugger = Debugger(dataset=opt.dataset,
ipynb=(opt.debug == 3),
theme=opt.debugger_theme)
img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * opt.std + opt.mean) * 255.), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hm')
debugger.add_blend_img(img, gt, 'gt_hm')
debugger.add_img(img, img_id='out_pred')
for k in range(len(dets[i])):
if dets[i, k, 4] > opt.center_thresh:
# add_coco_bbox(bbox, cat, conf=1, show_txt=True, img_id='default'):
debugger.add_coco_bbox(dets[i, k, :4],
dets[i, k, -1],
dets[i, k, 4],
img_id='out_pred')
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt[i])):
if dets_gt[i, k, 4] > opt.center_thresh:
# add_coco_bbox(bbox, cat, conf=1, show_txt=True, img_id='default'):
debugger.add_coco_bbox(dets_gt[i, k, :4],
dets_gt[i, k, -1],
dets_gt[i, k, 4],
img_id='out_gt')
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)
def run(self, image):
load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0
merge_time, tot_time = 0, 0
debugger = Debugger()
start_time = time.time()
detections = []
scale_start_time = time.time()
images, meta = self.pre_process(image)
images = images.cuda()
torch.cuda.synchronize()
pre_process_time = time.time()
pre_time += pre_process_time - scale_start_time
output, dets, forward_time = self.process(images, return_time=True)
torch.cuda.synchronize()
net_time += forward_time - pre_process_time
decode_time = time.time()
dec_time += decode_time - forward_time
dets = self.post_process(dets, meta, 1)
torch.cuda.synchronize()
post_process_time = time.time()
post_time += post_process_time - decode_time
detections.append(dets)
results = self.merge_outputs(detections)
torch.cuda.synchronize()
end_time = time.time()
merge_time += end_time - post_process_time
tot_time += end_time - start_time
self.show_results(debugger, image, results)
return {
'results': results,
'tot': tot_time,
'pre': pre_time,
'net': net_time,
'dec': dec_time,
'post': post_time,
'merge': merge_time
}
def demo_image(image, model, opt, ax):
s = max(image.shape[0], image.shape[1]) * 1.0
c = np.array([image.shape[1] / 2., image.shape[0] / 2.], dtype=np.float32)
trans_input = get_affine_transform(c, s, 0, [opt.input_w, opt.input_h])
inp = cv2.warpAffine(image,
trans_input, (opt.input_w, opt.input_h),
flags=cv2.INTER_LINEAR)
inp = (inp / 255. - mean) / std
inp = inp.transpose(2, 0, 1)[np.newaxis, ...].astype(np.float32)
inp = torch.from_numpy(inp).to(opt.device)
out = model(inp)[-1]
pred = get_preds(out['hm'].detach().cpu().numpy())[0]
pred = transform_preds(pred, c, s, (opt.output_w, opt.output_h))
# pred_3d = get_preds_3d(out['hm'].detach().cpu().numpy(),
# out['depth'].detach().cpu().numpy())[0]
preds, preds_3d, pred_3d = get_preds_3d_(
out['hm'].detach().cpu().numpy(), out['depth'].detach().cpu().numpy())
# print 3D points
print("The shape of pred_3d: \n", pred_3d.shape)
print("The shape of preds_3d_scr: \n", preds_3d.shape)
data_path = "/home/fa/Desktop/project/test1/pose_data_3d.txt"
with file(data_path, mode="a") as fw:
fw.write("# array shape: {0}\n".format(preds_3d.squeeze().shape))
# fw.write("# array: {0}\n".format("pred_3d"))
# np.savetxt(fw, np.squeeze(pred_3d), fmt="%-7.2f")
fw.write("# array: {0}\n".format("preds_3d"))
np.savetxt(fw, np.squeeze(preds_3d), fmt="%-7.2f")
debugger = Debugger()
debugger.add_img(image)
debugger.add_point_2d(pred, (255, 0, 0))
debugger.add_point_3d(pred_3d, ax, 'b')
debugger.show_all_imgs(pause=True)
debugger.show_3d(ax)
def run(self, image_or_path_or_tensor, meta=None):
load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0
merge_time, tot_time = 0, 0
debugger = Debugger(dataset=self.opt.dataset,
ipynb=(self.opt.debug == 3),
theme=self.opt.debugger_theme)
start_time = time.time()
pre_processed = False
if isinstance(image_or_path_or_tensor, np.ndarray):
image = image_or_path_or_tensor
elif type(image_or_path_or_tensor) == type(''):
imagename = os.path.basename(image_or_path_or_tensor) # 获取图片名
imagename = imagename.rsplit('.jpg')[0]
image = cv2.imread(image_or_path_or_tensor)
pos = image_or_path_or_tensor.rfind('.')
list_i = list(image_or_path_or_tensor) # str -> list
list_i.insert(pos, '_A') # 注意不用重新赋值
image2path = ''.join(list_i) # list -> str
image2 = cv2.imread(image2path)
else:
image = image_or_path_or_tensor['image'][0].numpy()
pre_processed_images = image_or_path_or_tensor
pre_processed = True
loaded_time = time.time()
load_time += (loaded_time - start_time)
detections = []
for scale in self.scales:
scale_start_time = time.time()
if not pre_processed:
images, meta = self.pre_process(image, scale, meta)
images2, meta = self.pre_process(image2, scale, meta)
else:
# import pdb; pdb.set_trace()
images = pre_processed_images['images'][scale][0]
meta = pre_processed_images['meta'][scale]
meta = {k: v.numpy()[0] for k, v in meta.items()}
images = images.to(self.opt.device)
images2 = images2.to(self.opt.device)
new_img = torch.cat((images, images2), 1) # 将两个tensor合为6通道
torch.cuda.synchronize()
pre_process_time = time.time()
pre_time += pre_process_time - scale_start_time
output, dets, forward_time = self.process(new_img,
return_time=True)
torch.cuda.synchronize()
net_time += forward_time - pre_process_time
decode_time = time.time()
dec_time += decode_time - forward_time
if self.opt.debug >= 2:
self.debug(debugger, images, dets, output, scale)
dets = self.post_process(dets, meta, scale)
torch.cuda.synchronize()
post_process_time = time.time()
post_time += post_process_time - decode_time
detections.append(dets)
results = self.merge_outputs(detections)
torch.cuda.synchronize()
end_time = time.time()
merge_time += end_time - post_process_time
tot_time += end_time - start_time
if self.opt.debug >= 1:
self.show_results(debugger, image, imagename, results)
return {
'results': results,
'tot': tot_time,
'load': load_time,
'pre': pre_time,
'net': net_time,
'dec': dec_time,
'post': post_time,
'merge': merge_time
}
def demo(opt):
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.debug = max(opt.debug, 1)
Detector = detector_factory[opt.task]
detector = Detector(opt)
video_output = None
print("FKKKKKK", opt.demo)
if opt.demo == 'webcam' or \
opt.demo[opt.demo.rfind('.') + 1:].lower() in video_ext:
cam = None
if opt.demo == 'webcam':
cam = cv2.VideoCapture(0, cv2.CAP_DSHOW)
else:
cam = cv2.VideoCapture(opt.demo)
width = int(cam.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cam.get(cv2.CAP_PROP_FRAME_HEIGHT))
csv_bf = None
csv_wr = None
if opt.save != '':
if not os.path.isdir("./output"):
os.mkdir("./output")
file_name = opt.demo
if opt.save == "all" or opt.save == "data":
csv_file_name = os.path.basename(
opt.demo).split('.')[0] + '.csv'
if opt.demo == 'webcam':
now = datetime.now()
current_time = now.strftime("%H_%M_%S %d_%m_%y")
file_name = current_time + '.mp4'
csv_file_name = current_time + '.csv'
saveMetaData(current_time, True)
else:
saveMetaData(opt.demo, False)
csv_bf = open(os.path.join("./output", csv_file_name),
'w',
newline='')
csv_wr = csv.writer(csv_bf, delimiter=',')
header = [
"index", "pitch", "yaw", "roll", "x", "y", "z",
"body_pitch", "body_yaw", "body_roll", "body_x", "body_y",
"body_z", "bb_x", "bb_y", "bb_w", "bb_h", "confidence"
]
for i in range(1, 18):
header.append("p" + str(i) + "_x")
header.append("p" + str(i) + "_y")
csv_wr.writerow(header)
if opt.save == "all" or opt.save == "video":
#fourcc = cv2.VideoWriter_fourcc(*'MP4V')
fps = 20 if opt.demo == "webcam" else cam.get(cv2.CAP_PROP_FPS)
fourcc = cv2.VideoWriter_fourcc(
*'MP4V'
) #if opt.demo == "webcam" else cam.get(cv2.CAP_PROP_FOURCC);
print("FOURCC", cam.get(cv2.CAP_PROP_FOURCC))
video_output = cv2.VideoWriter(
os.path.join("./output", file_name), fourcc, fps,
(width, height))
debugger = Debugger(dataset=opt.dataset,
ipynb=(opt.debug == 3),
theme=opt.debugger_theme,
out=video_output)
detector.pause = False
count = 0
condition = True if opt.demo == "webcam" else cam.isOpened()
while condition:
ret, img = cam.read()
if ret:
img = cv2.flip(img, 1)
orientation, position, borientation, bpos, keypoints = detector.run(
img, debugger)
if opt.wamp != '':
if orientation is not None and position is not None:
outputQueue.put(
(["CenterNet", position, orientation, []],
["CenterNet.Body", position, borientation, []]))
if csv_wr is not None:
output_list = [[count], orientation, position,
borientation, bpos, keypoints]
#print("output", output_list)
line = [
item for sublist in output_list for item in sublist
]
csv_wr.writerow(line)
if cv2.waitKey(1) == 113:
cam.release()
if video_output is not None:
video_output.release()
cv2.destroyAllWindows()
if csv_bf is not None:
csv_bf.close()
return # esc to quit
count += 1
else:
cam.release()
if video_output is not None:
video_output.release()
cv2.destroyAllWindows()
break
else:
if os.path.isdir(opt.demo):
image_names = []
ls = os.listdir(opt.demo)
for file_name in sorted(ls):
ext = file_name[file_name.rfind('.') + 1:].lower()
if ext in image_ext:
image_names.append(os.path.join(opt.demo, file_name))
else:
image_names = [opt.demo]
for (image_name) in image_names:
ret = detector.run(image_name)
time_str = ''
for stat in time_stats:
time_str = time_str + '{} {:.3f}s |'.format(stat, ret[stat])
print(time_str)
def step(split, epoch, opt, data_loader, model, optimizer=None):
if split == 'train':
model.train()
else:
model.eval()
crit = torch.nn.MSELoss()
acc_idxs = data_loader.dataset.acc_idxs
edges = data_loader.dataset.edges
shuffle_ref = data_loader.dataset.shuffle_ref
mean = data_loader.dataset.mean
std = data_loader.dataset.std
convert_eval_format = data_loader.dataset.convert_eval_format
Loss, Acc = AverageMeter(), AverageMeter()
data_time, batch_time = AverageMeter(), AverageMeter()
preds = []
nIters = len(data_loader)
bar = Bar('{}'.format(opt.exp_id), max=nIters)
end = time.time()
for i, batch in enumerate(data_loader):
data_time.update(time.time() - end)
input, target, meta = batch['input'], batch['target'], batch['meta']
input_var = input.cuda(device=opt.device, non_blocking=True)
target_var = target.cuda(device=opt.device, non_blocking=True)
output = model(input_var)
loss = crit(output[-1]['hm'], target_var)
for k in range(opt.num_stacks - 1):
loss += crit(output[k], target_var)
if split == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
input_ = input.cpu().numpy().copy()
input_[0] = flip(input_[0]).copy()[np.newaxis, ...]
input_flip_var = torch.from_numpy(input_).cuda(
device=opt.device, non_blocking=True)
output_flip = model(input_flip_var)
output_flip = shuffle_lr(
flip(output_flip[-1]['hm'].detach().cpu().numpy()[0]), shuffle_ref)
output_flip = output_flip.reshape(
1, opt.num_output, opt.output_h, opt.output_w)
# output_ = (output[-1].detach().cpu().numpy() + output_flip) / 2
output_flip = torch.from_numpy(output_flip).cuda(
device=opt.device, non_blocking=True)
output[-1]['hm'] = (output[-1]['hm'] + output_flip) / 2
pred, conf = get_preds(output[-1]['hm'].detach().cpu().numpy(), True)
preds.append(convert_eval_format(pred, conf, meta)[0])
Loss.update(loss.detach()[0], input.size(0))
Acc.update(accuracy(output[-1]['hm'].detach().cpu().numpy(),
target_var.detach().cpu().numpy(), acc_idxs))
batch_time.update(time.time() - end)
end = time.time()
if not opt.hide_data_time:
time_str = ' |Data {dt.avg:.3f}s({dt.val:.3f}s)' \
' |Net {bt.avg:.3f}s'.format(dt = data_time,
bt = batch_time)
else:
time_str = ''
Bar.suffix = '{split}: [{0}][{1}/{2}] |Total {total:} |ETA {eta:}' \
'|Loss {loss.avg:.5f} |Acc {Acc.avg:.4f}'\
'{time_str}'.format(epoch, i, nIters, total=bar.elapsed_td,
eta=bar.eta_td, loss=Loss, Acc=Acc,
split = split, time_str = time_str)
if opt.print_iter > 0:
if i % opt.print_iter == 0:
print('{}| {}'.format(opt.exp_id, Bar.suffix))
else:
bar.next()
if opt.debug >= 2:
gt = get_preds(target.cpu().numpy()) * 4
pred = get_preds(output[-1]['hm'].detach().cpu().numpy()) * 4
debugger = Debugger(ipynb=opt.print_iter > 0, edges=edges)
img = (input[0].numpy().transpose(1, 2, 0) * std + mean) * 256
img = img.astype(np.uint8).copy()
debugger.add_img(img)
debugger.add_mask(
cv2.resize(target[0].numpy().max(axis=0),
(opt.input_w, opt.input_h)), img, 'target')
debugger.add_mask(
cv2.resize(output[-1]['hm'][0].detach().cpu().numpy().max(axis=0),
(opt.input_w, opt.input_h)), img, 'pred')
debugger.add_point_2d(pred[0], (255, 0, 0))
debugger.add_point_2d(gt[0], (0, 0, 255))
debugger.show_all_imgs(pause=True)
bar.finish()
return {'loss': Loss.avg,
'acc': Acc.avg,
'time': bar.elapsed_td.total_seconds() / 60.}, preds
def run(self, image_or_path_or_tensor, meta=None):
load_time, pre_time, net_time, dec_time, post_time = 0, 0, 0, 0, 0
merge_time, tot_time = 0, 0
debugger = Debugger(dataset=self.opt.dataset,
ipynb=(self.opt.debug == 3),
theme=self.opt.debugger_theme)
start_time = time.time()
pre_processed = False
if isinstance(image_or_path_or_tensor, np.ndarray):
image = image_or_path_or_tensor
elif type(image_or_path_or_tensor) == type(''):
image = cv2.imread(image_or_path_or_tensor)
else:
image = image_or_path_or_tensor['image'][0].numpy()
pre_processed_images = image_or_path_or_tensor
pre_processed = True
loaded_time = time.time()
load_time += (loaded_time - start_time)
detections = []
for scale in self.scales:
scale_start_time = time.time()
if not pre_processed:
images, meta = self.pre_process(image, scale, meta)
else:
# import pdb; pdb.set_trace()
images = pre_processed_images['images'][scale][0]
meta = pre_processed_images['meta'][scale]
meta = {k: v.numpy()[0] for k, v in meta.items()}
images = images.to(self.opt.device)
torch.cuda.synchronize()
pre_process_time = time.time()
pre_time += pre_process_time - scale_start_time
output, dets, forward_time = self.process(images, return_time=True)
torch.cuda.synchronize()
net_time += forward_time - pre_process_time
decode_time = time.time()
dec_time += decode_time - forward_time
if self.opt.debug >= 2:
self.debug(debugger, images, dets, output, scale)
dets = self.post_process(dets, meta, scale)
torch.cuda.synchronize()
post_process_time = time.time()
post_time += post_process_time - decode_time
detections.append(dets)
results = self.merge_outputs(detections)
torch.cuda.synchronize()
end_time = time.time()
merge_time += end_time - post_process_time
tot_time += end_time - start_time
if self.opt.debug >= 1:
self.show_results(debugger, image, results)
return {
'results': results,
'tot': tot_time,
'load': load_time,
'pre': pre_time,
'net': net_time,
'dec': dec_time,
'post': post_time,
'merge': merge_time
}
def debug(self, batch, output, iter_id):
opt = self.opt
reg = output['reg'] if opt.reg_offset else None
hm_hp = output['hm_hp'] if opt.hm_hp else None
hp_offset = output['hp_offset'] if opt.reg_hp_offset else None
dets = multi_pose_decode(output['hm'],
output['wh'],
output['hps'],
reg=reg,
hm_hp=hm_hp,
hp_offset=hp_offset,
K=opt.K)
dets = dets.detach().cpu().numpy().reshape(1, -1, dets.shape[2])
dets[:, :, :4] *= opt.input_res / opt.output_res
dets[:, :, 5:39] *= opt.input_res / opt.output_res
dets_gt = batch['meta']['gt_det'].numpy().reshape(1, -1, dets.shape[2])
dets_gt[:, :, :4] *= opt.input_res / opt.output_res
dets_gt[:, :, 5:39] *= opt.input_res / opt.output_res
for i in range(1):
debugger = Debugger(dataset=opt.dataset,
ipynb=(opt.debug == 3),
theme=opt.debugger_theme)
img = batch['input'][i].detach().cpu().numpy().transpose(1, 2, 0)
img = np.clip(((img * opt.std + opt.mean) * 255.), 0,
255).astype(np.uint8)
pred = debugger.gen_colormap(
output['hm'][i].detach().cpu().numpy())
gt = debugger.gen_colormap(batch['hm'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hm')
debugger.add_blend_img(img, gt, 'gt_hm')
debugger.add_img(img, img_id='out_pred')
for k in range(len(dets[i])):
if dets[i, k, 4] > opt.center_thresh:
debugger.add_coco_bbox(dets[i, k, :4],
dets[i, k, -1],
dets[i, k, 4],
img_id='out_pred')
debugger.add_coco_hp(dets[i, k, 5:39], img_id='out_pred')
debugger.add_img(img, img_id='out_gt')
for k in range(len(dets_gt[i])):
if dets_gt[i, k, 4] > opt.center_thresh:
debugger.add_coco_bbox(dets_gt[i, k, :4],
dets_gt[i, k, -1],
dets_gt[i, k, 4],
img_id='out_gt')
debugger.add_coco_hp(dets_gt[i, k, 5:39], img_id='out_gt')
if opt.hm_hp:
pred = debugger.gen_colormap_hp(
output['hm_hp'][i].detach().cpu().numpy())
gt = debugger.gen_colormap_hp(
batch['hm_hp'][i].detach().cpu().numpy())
debugger.add_blend_img(img, pred, 'pred_hmhp')
debugger.add_blend_img(img, gt, 'gt_hmhp')
if opt.debug == 4:
debugger.save_all_imgs(opt.debug_dir,
prefix='{}'.format(iter_id))
else:
debugger.show_all_imgs(pause=True)