def accuracy(output, target, hm_type='gaussian', thr=0.5):
'''
Calculate accuracy according to PCK,
but uses ground truth heatmap rather than x,y locations
First value to be returned is average accuracy across 'idxs',
followed by individual accuracies
'''
idx = list(range(output.shape[1])) # idx为特征点的编号
norm = 1.0
if hm_type == 'gaussian':
pred, _ = get_max_preds(output)
target, _ = get_max_preds(target)
h = output.shape[2]
w = output.shape[3]
norm = np.ones((pred.shape[0], 2)) * np.array([h, w]) / 10 # 32 * 2上，每个点的值为1 * 64 /10
dists = calc_dists(pred, target, norm) # 在heatmap（64 * 64）上计算预测值和GT值之间的差距，5 * 32维度
acc = np.zeros((len(idx) + 1))
avg_acc = 0
cnt = 0
for i in range(len(idx)):
acc[i + 1] = dist_acc(dists[idx[i]]) # 计算dists矩阵32张图中，heatmap上，某一特征点与GT值距离小于0.5的图像个数 / 一个batch中特征点有效的img张数
if acc[i + 1] >= 0:
avg_acc = avg_acc + acc[i + 1]
cnt += 1
avg_acc = avg_acc / cnt if cnt != 0 else 0 # 计算平均误差率
if cnt != 0:
acc[0] = avg_acc
return acc, avg_acc, cnt, pred
def accuracy(output, target, hm_type='gaussian', thr=0.5):
idx = list(range(output.shape[1]))
norm = 1.0
if hm_type == 'gaussian':
pred, _ = get_max_preds(output)
target, _ = get_max_preds(target)
h = output.shape[2]
w = output.shape[3]
norm = np.ones((pred.shape[0], 2)) * np.array([h, w]) / 10
dists = calc_dists(pred, target, norm)
acc = np.zeros((len(idx) + 1))
avg_acc = 0
cnt = 0
for i in range(len(idx)):
acc[i + 1] = dist_acc(dists[idx[i]], thr)
if acc[i + 1] >= 0:
avg_acc = avg_acc + acc[i + 1]
cnt += 1
avg_acc = avg_acc / cnt if cnt != 0 else 0
if cnt != 0:
acc[0] = avg_acc
return acc, avg_acc, cnt, pred
def accuracy(output, target, hm_type='gaussian', thr=0.5):
'''
Calculate accuracy according to PCK,
but uses ground truth heatmap rather than x,y locations
First value to be returned is average accuracy across 'idxs',
followed by individual accuracies
'''
idx = list(range(output.shape[1]))
norm = 1.0
if hm_type == 'gaussian':
pred, _ = get_max_preds(output)
target, _ = get_max_preds(target)
h = output.shape[2]
w = output.shape[3]
norm = np.ones((pred.shape[0], 2)) * np.array([h, w]) / 10
dists = calc_dists(pred, target, norm)
acc = np.zeros((len(idx) + 1))
avg_acc = 0
cnt = 0
for i in range(len(idx)):
acc[i + 1] = dist_acc(dists[idx[i]])
if acc[i + 1] >= 0:
avg_acc = avg_acc + acc[i + 1]
cnt += 1
avg_acc = avg_acc / cnt if cnt != 0 else 0
if cnt != 0:
acc[0] = avg_acc
return acc, avg_acc, cnt, pred
def accuracy(output, target, hm_type='gaussian', thr=0.5, input_shape=None):
'''
Calculate accuracy according to PCK,
but uses ground truth heatmap rather than x,y locations
First value to be returned is average accuracy across 'idxs',
followed by individual accuracies
'''
idx = list(range(output.shape[1]))
norm = 1.0
if output.ndim == 4:
pred, _ = get_max_preds(output, input_shape)
target, _ = get_max_preds(target, input_shape)
h = output.shape[2]
w = output.shape[3]
norm = np.ones((pred.shape[0], 2)) * np.array([h, w]) / 10
dists = calc_dists(pred, target, norm)
else:
pred = get_rotation_matrix_to_decart(output, input_shape)
target = get_rotation_matrix_to_decart(target, input_shape)
dists = np.zeros((pred.shape[1], pred.shape[0]))
norm = np.array([input_shape[1], input_shape[0]]) / 10
for n in range(pred.shape[0]):
for c in range(pred.shape[1]):
if target[n, c, 0] > 1 and target[n, c, 1] > 1:
dists[c, n] = np.linalg.norm(pred[n, c, :] / norm -
target[n, c, :] / norm)
else:
dists[c, n] = -1
# dists = calc_dists(preds, target, norm)
pred *= 64 / np.array([input_shape[1], input_shape[0]])
acc = np.zeros((len(idx) + 1))
avg_acc = 0
cnt = 0
for i in range(len(idx)):
acc[i + 1] = dist_acc(dists[idx[i]])
if acc[i + 1] >= 0:
avg_acc = avg_acc + acc[i + 1]
cnt += 1
avg_acc = avg_acc / cnt if cnt != 0 else 0
if cnt != 0:
acc[0] = avg_acc
return acc, avg_acc, cnt, pred
def save_all_heatmaps(
batch_image,
batch_heatmaps,
output_dir,
meta,
normalize=True,
):
'''
Need test batch size =1
batch_image: [batch_size, channel, height, width]
batch_heatmaps: ['batch_size, num_joints, height, width]
file_name: saved file name
'''
heatmaps_dir = os.path.join(output_dir, 'heatmaps')
if not os.path.exists(heatmaps_dir):
os.makedirs(heatmaps_dir, exist_ok=True)
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_size = batch_heatmaps.size(0)
num_joints = batch_heatmaps.size(1)
heatmap_height = batch_heatmaps.size(2)
heatmap_width = batch_heatmaps.size(3)
heatmap_image = np.zeros((heatmap_height, heatmap_width, 3))
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
for i in range(batch_size):
image = batch_image[i].mul(255)\
.clamp(0, 255)\
.byte()\
.permute(1, 2, 0)\
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
height_begin = heatmap_height * i
height_end = heatmap_height * (i + 1)
img_name = ((meta['image'][i]).split("/")[-1]).split(".")[0]
print(img_name)
for j in range(num_joints):
img_dir = os.path.join(heatmaps_dir, img_name)
if not os.path.exists(img_dir):
os.mkdir(img_dir)
heatmap_filepath = os.path.join(
img_dir, (img_name + '_heatmap_' + str(j) + ".jpg"))
heatmap = heatmaps[j, :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
cv2.imwrite(heatmap_filepath, colored_heatmap)
def main():
args = parse_args()
update_config(cfg, args)
logger, final_output_dir, tb_log_dir = create_logger(
cfg, args.cfg, 'valid')
exec_net, net_input_shape = load_to_IE(args.model)
# We need dynamically generated key for fetching output tensor
output_key = list(exec_net.outputs.keys())[0]
# Data loading code
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
)
valid_dataset = eval('dataset.'+cfg.DATASET.DATASET)(
cfg, cfg.DATASET.ROOT, cfg.DATASET.TEST_SET, False,
transforms.Compose([
transforms.ToTensor(),
normalize,
])
)
valid_loader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=1,
shuffle=False,
num_workers=cfg.WORKERS,
pin_memory=True
)
process_time = AverageMeter()
with torch.no_grad():
for i, (input, target, target_weight, meta) in enumerate(valid_loader):
start_time = time.time()
# compute output
output = sync_inference(exec_net, image=np.expand_dims(input[0].numpy(), 0))
batch_heatmaps = output[output_key]
coords, maxvals = get_max_preds(batch_heatmaps)
# measure elapsed time
process_time.update(time.time() - start_time)
prefix = '{}_{}'.format(
os.path.join(final_output_dir, 'val'), i
)
save_debug_images(cfg, input, meta, target, coords * 4, torch.from_numpy(batch_heatmaps),
prefix)
if i == 100:
break
logger.info(f'OpenVINO IE: Inference EngineAverage processing time of model:{process_time.avg}')
def test(config, input, model, output_dir, idx):
# switch to evaluate mode
model.eval()
with torch.no_grad():
output = model(input)
pred, _ = get_max_preds(output.cpu().numpy())
prefix = '{}_{}'.format(os.path.join(output_dir, 'test'), idx)
save_debug_images_test(config, input, pred * 4, output, prefix)
def visualize(config, image_names, images, heatmaps, output_dir):
# get size
batch_size, num_joints, heatmap_h, heatmap_w = heatmaps.shape
# get keypoints
coords, maxvals = get_max_preds(heatmaps)
# save heatmap
for i in range(batch_size):
resized_heatmap = resize_heatmap(heatmaps[i], 18, 24)
with open(os.path.join(output_dir, image_names[i]+'.pkl'), 'wb') as f:
cPickle.dump(resized_heatmap, f, protocol=cPickle.HIGHEST_PROTOCOL)
def pred(output, hm_type='gaussian', thr=0.5):
'''
Calculate accuracy according to PCK,
but uses ground truth heatmap rather than x,y locations
First value to be returned is average accuracy across 'idxs',
followed by individual accuracies
'''
idx = list(range(output.shape[1]))
norm = 1.0
if hm_type == 'gaussian':
pred, _ = get_max_preds(output) #core.inference
return pred
def get_final_preds(config, batch_heatmaps):
heatmap_height = batch_heatmaps.shape[2]
heatmap_width = batch_heatmaps.shape[3]
# post-processing
if config.TEST.POST_PROCESS:
preds, maxval = get_max_preds(batch_heatmaps)
# Transform back
# for i in range(preds.shape[0]):
# preds[i] = transform_preds(preds[i], center[i], scale[i],
# [heatmap_width, heatmap_height])
return preds, maxval
def save_prediction(config, input, output, prefix, metas):
fname = f'{prefix}.json'
batch_size = output.size(0)
num_joints = output.size(1)
heatmap_height = output.size(2)
heatmap_width = output.size(3)
preds, maxvals = get_max_preds(output.detach().cpu().numpy())
d = {}
for i in range(batch_size):
batch_key = f'batch_{i}'
fpath = metas['path'][i]
im_width = metas['width'].detach().cpu().numpy()[i]
im_height = metas['height'].detach().cpu().numpy()[i]
d[batch_key] = {}
d[batch_key]['pred'] = {
'width': heatmap_width,
'height': heatmap_height,
'joints': {}
}
d[batch_key]['input'] = {
'path': fpath,
'width': im_width,
'height': im_height
}
for j in range(num_joints):
joint_key = f'joint_{j}'
x = int(preds[i][j][0])
y = int(preds[i][j][1])
d[batch_key]['pred']['joints'][joint_key] = [x, y]
with open(fname, 'w') as f:
dump(d, f, indent=2)
return d
def evaluate(config, val_loader, val_dataset, model, output_dir):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
num_samples = len(val_dataset)
all_preds = np.zeros((num_samples, config.MODEL.NUM_JOINTS, 3),
dtype=np.float32)
all_boxes = np.zeros((num_samples, 6))
image_path = []
filenames = []
imgnums = []
idx = 0
with torch.no_grad():
for i, (input, _, _, meta) in enumerate(val_loader):
# compute output
output = model(input)
pred, _ = get_max_preds(output.cpu().numpy())
image_path.extend(meta['image'])
if config.DATASET.DATASET == 'posetrack':
filenames.extend(meta['filename'])
imgnums.extend(meta['imgnum'].numpy())
prefix = '{}_{}'.format(os.path.join(output_dir, 'val'), i)
if 'OUT_FILE' in config:
prefix = os.path.join(output_dir, config.OUT_FILE)
save_output_images(config, input, meta, pred*4, prefix)
# TODO save output points for analysis
return
def save_batch_heatmaps(batch_image, batch_heatmaps, file_name,
normalize=True):
'''
batch_image: [batch_size, channel, height, width]
batch_heatmaps: ['batch_size, num_joints, height, width]
file_name: saved file name
'''
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_size = batch_heatmaps.size(0)
num_joints = batch_heatmaps.size(1)
heatmap_height = batch_heatmaps.size(2)
heatmap_width = batch_heatmaps.size(3)
grid_image = np.zeros((batch_size*heatmap_height,
(num_joints+1)*heatmap_width,
3),
dtype=np.uint8)
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
for i in range(batch_size):
image = batch_image[i].mul(255)\
.clamp(0, 255)\
.byte()\
.permute(1, 2, 0)\
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
resized_image = cv2.resize(image,
(int(heatmap_width), int(heatmap_height)))
height_begin = heatmap_height * i
height_end = heatmap_height * (i + 1)
for j in range(num_joints):
cv2.circle(resized_image,
(int(preds[i][j][0]), int(preds[i][j][1])),
1, [0, 0, 255], 1)
heatmap = heatmaps[j, :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
masked_image = colored_heatmap*0.7 + resized_image*0.3
cv2.circle(masked_image,
(int(preds[i][j][0]), int(preds[i][j][1])),
1, [0, 0, 255], 1)
width_begin = heatmap_width * (j+1)
width_end = heatmap_width * (j+2)
grid_image[height_begin:height_end, width_begin:width_end, :] = \
masked_image
# grid_image[height_begin:height_end, width_begin:width_end, :] = \
# colored_heatmap*0.7 + resized_image*0.3
grid_image[height_begin:height_end, 0:heatmap_width, :] = resized_image
cv2.imwrite(file_name, grid_image)
def save_batch_heatmaps(batch_image, batch_heatmaps, file_name,
normalize=True, MV_IDEA=False):
# visualize only the first sample
batch_image = batch_image[:1, :, :, :]
batch_heatmaps = batch_heatmaps[:1, :]
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_size = batch_heatmaps.size(0)
num_joints = batch_heatmaps.size(1)
heatmap_height = batch_heatmaps.size(2)
heatmap_width = batch_heatmaps.size(3)
grid_image = np.zeros((batch_size*heatmap_height,
2*heatmap_width,
3),
dtype=np.uint8)
if MV_IDEA:
preds = get_heatmap_center_preds(batch_heatmaps.detach().cpu().numpy())
else:
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
for i in range(batch_size):
image = batch_image[i].mul(255)\
.clamp(0, 255)\
.byte()\
.permute(1, 2, 0)\
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
resized_image = cv2.resize(image,
(int(heatmap_width), int(heatmap_height)))
height_begin = heatmap_height * i
height_end = heatmap_height * (i + 1)
cv2.circle(resized_image,
(int(preds[i][0][0]), int(preds[i][0][1])),
1, [0, 0, 255], 1)
heatmap = heatmaps[0, :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
masked_image = colored_heatmap*0.7 + resized_image*0.3
cv2.circle(masked_image,
(int(preds[i][0][0]), int(preds[i][0][1])),
1, [0, 0, 255], 1)
width_begin = heatmap_width
width_end = heatmap_width * 2
grid_image[height_begin:height_end, width_begin:width_end, :] = \
masked_image
grid_image[height_begin:height_end, 0:heatmap_width, :] = resized_image
cv2.imwrite(file_name, grid_image)
def save_batch_paf_maps(batch_image,
batch_heatmaps,
batch_paf_maps,
file_name,
normalize=True):
'''
batch_image: [batch_size, channel, height, width]
batch_paf_maps: [batch_size, num_pairs, height, width]
file_name: saved file name
'''
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_size = batch_paf_maps.size(0)
num_pairs = batch_paf_maps.size(1) // 2
paf_map_height = batch_paf_maps.size(2)
paf_map_width = batch_paf_maps.size(3)
grid_image = np.zeros(
(batch_size * paf_map_height, (num_pairs + 1) * paf_map_width, 3),
dtype=np.uint8)
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
joint_pairs = [[1, 8], [8, 9], [9, 10], [1, 11], [11, 12], [12, 13],
[1, 2], [2, 3], [3, 4], [2, 16], [1, 5], [5, 6], [6, 7],
[5, 17], [1, 0], [0, 14], [0, 15], [14, 16], [15, 17]]
for i in range(batch_size):
image = batch_image[i].mul(255)\
.clamp(0, 255)\
.byte()\
.permute(1, 2, 0)\
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
paf_maps = batch_paf_maps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
resized_image = cv2.resize(
image, # 变到map的尺寸
(int(paf_map_width), int(paf_map_height)))
height_begin = paf_map_height * i
height_end = paf_map_height * (i + 1)
for j in range(num_pairs):
cv2.circle(resized_image, (int(preds[i][joint_pairs[j][0]][0]),
int(preds[i][joint_pairs[j][0]][1])), 1,
[0, 0, 255], 1)
cv2.circle(resized_image, (int(preds[i][joint_pairs[j][1]][0]),
int(preds[i][joint_pairs[j][1]][1])), 1,
[0, 0, 255], 1)
for k in range(paf_map_height):
for l in range(paf_map_width):
# keypoint_b = joints[self.joint_pairs[paf_id][1]-1]
if batch_paf_maps[i][
2 * j][k][l] != 0: # 说明改点=v，以该点为起点画一个v（单位向量）
# if heatmaps[2*j][k][l] != 0: # 这样应该也行
# 这里v的两个坐标都是小数，变成整数后和出发点相同了，画不出来
# 向上取整可以吗？
# cv2.line(resized_image,
# (l, k),
# (l+round(batch_paf_maps[i][2*j][k][l]), k+round(batch_paf_maps[i][2*j+1][k][l])),
# (0, 255, 0), 1)
# cv2.circle(resized_image, (k, l), 1, (255, 0, 0), 1) # 点点都糊成一团了
pass
heatmap = np.zeros_like(heatmaps[0], dtype=np.uint8)
heatmap2 = np.zeros_like(heatmaps[0], dtype=np.uint8)
if np.max(paf_maps[2 * j, :, :]) > 0:
heatmap = heatmaps[joint_pairs[j][0], :, :]
heatmap2 = heatmaps[joint_pairs[j][1], :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
colored_heatmap2 = cv2.applyColorMap(heatmap2, cv2.COLORMAP_JET)
paf_map = paf_maps[2 * j, :, :] # 这里对吗？
colored_paf_map = cv2.applyColorMap(
paf_map, cv2.COLORMAP_JET) # 0就是(128, 0, 0)了，就是蓝色的
masked_image = colored_heatmap*0.7 + colored_heatmap2*0.7 \
+ colored_paf_map*0.7 + resized_image*0.3
for k in range(paf_map_height):
for l in range(paf_map_width):
# keypoint_b = joints[self.joint_pairs[paf_id][1]-1]
if batch_paf_maps[i][
2 * j][k][l] != 0: # 说明改点=v，以该点为起点画一个v（单位向量）
# cv2.line(masked_image,
# (l, k),
# (l+round(batch_paf_maps[i][2*j][k][l]), k+round(batch_paf_maps[i][2*j+1][k][l])),
# (0, 255, 0), 1)
# cv2.circle(masked_image, (k, l), 1, (255, 0, 0), 1)
pass
width_begin = paf_map_width * (j + 1)
width_end = paf_map_width * (j + 2)
grid_image[height_begin:height_end, width_begin:width_end, :] = \
masked_image
grid_image[height_begin:height_end,
0:paf_map_width, :] = resized_image # 第一列是放原图的
cv2.imwrite(file_name, grid_image)
def validate(config,
val_loader,
val_dataset,
model,
criterion,
output_dir,
tb_log_dir,
epoch,
writer_dict=None):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
num_samples = len(val_dataset)
all_preds = np.zeros((num_samples, config.MODEL.NUM_JOINTS, 3),
dtype=np.float32)
all_boxes = np.zeros((num_samples, 6))
image_path = []
filenames = []
imgnums = []
idx = 0
logger.info(f'# VALIDATE: EPOCH {epoch}')
model = add_flops_counting_methods(model)
model.start_flops_count()
model.eval()
flops_per_layer = []
total_per_layer = []
with torch.no_grad():
end = time.time()
val_iter = val_loader.__iter__()
num_step = len(val_iter)
for i in range(num_step):
input, target, target_weight, meta = next(val_iter)
input = input.to('cuda', non_blocking=True)
dynconv_meta = make_dynconv_meta(config, epoch, i)
outputs, dynconv_meta = model(input, dynconv_meta)
if 'masks' in dynconv_meta:
percs, cost, total = dynconv.cost_per_layer(dynconv_meta)
flops_per_layer.append(cost)
total_per_layer.append(total)
output = outputs[-1] if isinstance(outputs, list) else outputs
# if config.TEST.FLIP_TEST:
# flip not supported for dynconv
# # this part is ugly, because pytorch has not supported negative index
# # input_flipped = model(input[:, :, :, ::-1])
# input_flipped = np.flip(input.cpu().numpy(), 3).copy()
# input_flipped = torch.from_numpy(input_flipped).cuda()
# outputs_flipped = model(input_flipped)
# if isinstance(outputs_flipped, list):
# output_flipped = outputs_flipped[-1]
# else:
# output_flipped = outputs_flipped
# output_flipped = flip_back(output_flipped.cpu().numpy(),
# val_dataset.flip_pairs)
# output_flipped = torch.from_numpy(output_flipped.copy()).cuda()
# # feature is not aligned, shift flipped heatmap for higher accuracy
# if config.TEST.SHIFT_HEATMAP:
# output_flipped[:, :, :, 1:] = \
# output_flipped.clone()[:, :, :, 0:-1]
# output = (output + output_flipped) * 0.5
target = target.cuda(non_blocking=True)
target_weight = target_weight.cuda(non_blocking=True)
loss = criterion(output, target, target_weight)
num_images = input.size(0)
# measure accuracy and record loss
losses.update(loss.item(), num_images)
_, avg_acc, cnt, pred = accuracy(output.cpu().numpy(),
target.cpu().numpy())
acc.update(avg_acc, cnt)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
c = meta['center'].numpy()
s = meta['scale'].numpy()
score = meta['score'].numpy()
output_np = output.clone().cpu().numpy()
preds_rel, maxvals_rel = get_max_preds(output_np)
preds, maxvals = get_final_preds(config, output_np, c, s)
all_preds[idx:idx + num_images, :, 0:2] = preds[:, :, 0:2]
all_preds[idx:idx + num_images, :, 2:3] = maxvals
# double check this all_boxes parts
all_boxes[idx:idx + num_images, 0:2] = c[:, 0:2]
all_boxes[idx:idx + num_images, 2:4] = s[:, 0:2]
all_boxes[idx:idx + num_images, 4] = np.prod(s * 200, 1)
all_boxes[idx:idx + num_images, 5] = score
image_path.extend(meta['image'])
idx += num_images
if i % config.PRINT_FREQ == 0:
msg = 'Test: [{0}/{1}]\t' \
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \
'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \
'Accuracy {acc.val:.3f} ({acc.avg:.3f})'.format(
i, len(val_loader), batch_time=batch_time,
loss=losses, acc=acc)
logger.info(msg)
prefix = '{}_{}'.format(os.path.join(output_dir, 'val'), i)
save_debug_images(config, input, meta, target, pred * 4,
output, prefix)
if config.DEBUG.PONDER:
img = viz.frame2mpl(input[0], denormalize=True)
img = viz.add_skeleton(img,
preds_rel[0] * 4,
maxvals_rel[0],
thres=0.2)
plt.figure()
plt.title('input')
plt.imshow(img)
ponder_cost = dynconv.ponder_cost_map(dynconv_meta['masks'])
if ponder_cost is not None:
plt.figure()
plt.title('ponder cost map')
plt.imshow(ponder_cost,
vmin=2,
vmax=len(dynconv_meta['masks']) - 2)
plt.colorbar()
else:
logger.info('Not a sparse model - no ponder cost')
viz.showKey()
name_values, perf_indicator = val_dataset.evaluate(
config, all_preds, output_dir, all_boxes, image_path, filenames,
imgnums)
model_name = config.MODEL.NAME
if isinstance(name_values, list):
for name_value in name_values:
_print_name_value(name_value, model_name)
else:
_print_name_value(name_values, model_name)
if writer_dict:
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar('valid_loss', losses.avg, global_steps)
writer.add_scalar('valid_acc', acc.avg, global_steps)
if isinstance(name_values, list):
for name_value in name_values:
writer.add_scalars('valid', dict(name_value), global_steps)
else:
writer.add_scalars('valid', dict(name_values), global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
avg_flops, total_flops, batch_count = model.compute_average_flops_cost()
logger.info(
f'# PARAMS: {get_model_parameters_number(model, as_string=False)/1e6} M'
)
logger.info(
f'# FLOPS (multiply-accumulates, MACs): {(total_flops/idx)/1e9} GMacs on {idx} images'
)
# some conditional execution statistics
if len(flops_per_layer) > 0:
flops_per_layer = torch.cat(flops_per_layer, dim=0)
total_per_layer = torch.cat(total_per_layer, dim=0)
perc_per_layer = flops_per_layer / total_per_layer
perc_per_layer_avg = perc_per_layer.mean(dim=0)
perc_per_layer_std = perc_per_layer.std(dim=0)
s = ''
for perc in perc_per_layer_avg:
s += f'{round(float(perc), 2)}, '
logger.info(
f'# FLOPS (multiply-accumulates MACs) used percentage per layer (average): {s}'
)
s = ''
for std in perc_per_layer_std:
s += f'{round(float(std), 2)}, '
logger.info(
f'# FLOPS (multiply-accumulates MACs) used percentage per layer (standard deviation): {s}'
)
exec_cond_flops = int(torch.sum(flops_per_layer)) / idx
total_cond_flops = int(torch.sum(total_per_layer)) / idx
logger.info(
f'# Conditional FLOPS (multiply-accumulates MACs) over all layers (average per image): {exec_cond_flops/1e9} GMac out of {total_cond_flops/1e9} GMac ({round(100*exec_cond_flops/total_cond_flops,1)}%)'
)
return perf_indicator
def evaluate(config,
val_loader,
val_dataset,
model,
criterion,
output_dir,
tb_log_dir,
args,
writer_dict=None):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
num_samples = len(val_dataset)
all_preds = np.zeros((num_samples, config.MODEL.NUM_JOINTS, 3),
dtype=np.float32)
all_boxes = np.zeros((num_samples, 6))
image_path = []
filenames = []
imgnums = []
idx = 0
with torch.no_grad():
end = time.time()
for i, (input, target, target_weight, meta) in enumerate(val_loader):
# compute output
outputs = model(input)
if isinstance(outputs, list):
output = outputs[-1]
else:
output = outputs
if config.TEST.FLIP_TEST:
print('flippin')
# this part is ugly, because pytorch has not supported negative index
# input_flipped = model(input[:, :, :, ::-1])
input_flipped = np.flip(input.cpu().numpy(), 3).copy()
input_flipped = torch.from_numpy(input_flipped).cuda()
outputs_flipped = model(input_flipped)
if isinstance(outputs_flipped, list):
output_flipped = outputs_flipped[-1]
else:
output_flipped = outputs_flipped
output_flipped = flip_back(output_flipped.cpu().numpy(),
val_dataset.flip_pairs)
output_flipped = torch.from_numpy(output_flipped.copy()).cuda()
# feature is not aligned, shift flipped heatmap for higher accuracy
if config.TEST.SHIFT_HEATMAP:
output_flipped[:, :, :, 1:] = \
output_flipped.clone()[:, :, :, 0:-1]
output = (output + output_flipped) * 0.5
pred, _ = get_max_preds(output.cpu().numpy())
image_path.extend(meta['image'])
num_images = input.size(0)
idx += num_images
print(output_dir)
prefix = '{}_{}'.format(os.path.join(output_dir, 'val'), i)
# save_debug_images(config, input, meta, target, pred*4, output, prefix)
save_output_images(config, input, meta, pred * 4, output, prefix,
args)
return
def main():
args = parse_args()
update_config(cfg, args)
logger, final_output_dir, tb_log_dir = create_logger(
cfg, args.cfg, 'valid')
model = eval('models.'+cfg.MODEL.NAME+'.get_pose_net')(
cfg, is_train=False
)
logger.info('=> loading model from {}'.format(cfg.TEST.MODEL_FILE))
model.load_state_dict(torch.load(cfg.TEST.MODEL_FILE, map_location=torch.device('cpu')), strict=False)
# Data loading code
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
)
valid_dataset = eval('dataset.'+cfg.DATASET.DATASET)(
cfg, cfg.DATASET.ROOT, cfg.DATASET.TEST_SET, False,
transforms.Compose([
transforms.ToTensor(),
normalize,
])
)
valid_loader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=1,
shuffle=False,
num_workers=cfg.WORKERS,
pin_memory=True
)
process_time = AverageMeter()
# switch to evaluate mode
model.eval()
if args.convert_onnx:
x_tensor = torch.rand(1, 3, 256, 192)
torch.onnx.export(model.cpu(), x_tensor.cpu(), 'model.onnx', export_params=True,
operator_export_type=torch.onnx.OperatorExportTypes.ONNX,
opset_version=9,
verbose=False)
logger.info('Model is converted to ONNX')
with torch.no_grad():
for i, (input, target, target_weight, meta) in enumerate(valid_loader):
start_time = time.time()
# compute output
output = model(input)
batch_heatmaps = output.clone().cpu().numpy()
coords, maxvals = get_max_preds(batch_heatmaps)
# measure elapsed time
process_time.update(time.time() - start_time)
prefix = '{}_{}'.format(
os.path.join(final_output_dir, 'val'), i
)
save_debug_images(cfg, input, meta, target, coords * 4, output,
prefix)
if i == 100:
break
logger.info(f'PyTorch: Inference EngineAverage processing time of model:{process_time.avg}')
def validate(config, val_loader, val_dataset, model, criterion, output_dir,
tb_log_dir, writer_dict=None):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
num_samples = len(val_dataset)
all_preds = np.zeros(
(num_samples, config.MODEL.NUM_JOINTS, 3),
dtype=np.float32
)
all_boxes = np.zeros((num_samples, 6))
image_path = []
filenames = []
imgnums = []
idx = 0
export_annots = []
target_weights = []
pred_max_vals_valid = []
with torch.no_grad():
end = time.time()
for i, (input, target, target_weight, meta) in enumerate(val_loader):
# compute output
outputs = model(input)
if isinstance(outputs, list):
output = outputs[-1]
else:
output = outputs
target_weights.append(target_weight)
if config.TEST.FLIP_TEST:
# this part is ugly, because pytorch has not supported negative index
# input_flipped = model(input[:, :, :, ::-1])
input_flipped = np.flip(input.cpu().numpy(), 3).copy()
input_flipped = torch.from_numpy(input_flipped)
if config.USE_GPU:
input_flipped = input_flipped.cuda()
outputs_flipped = model(input_flipped)
if isinstance(outputs_flipped, list):
output_flipped = outputs_flipped[-1]
else:
output_flipped = outputs_flipped
output_flipped = flip_back(output_flipped.cpu().numpy(),
val_dataset.flip_pairs)
output_flipped = torch.from_numpy(output_flipped.copy())
if config.USE_GPU:
output_flipped = output_flipped.cuda()
# feature is not aligned, shift flipped heatmap for higher accuracy
if config.TEST.SHIFT_HEATMAP:
output_flipped[:, :, :, 1:] = \
output_flipped.clone()[:, :, :, 0:-1]
output = (output + output_flipped) * 0.5
if config.USE_GPU:
target = target.cuda(non_blocking=True)
target_weight = target_weight.cuda(non_blocking=True)
loss = criterion(output, target, target_weight)
num_images = input.size(0)
# measure accuracy and record loss
losses.update(loss.item(), num_images)
_, avg_acc, cnt, pred = accuracy(output.cpu().numpy(),
target.cpu().numpy())
pred_maxvals = get_max_preds(output.cpu().numpy())[1]
acc.update(avg_acc, cnt)
pred_max_vals_valid.append(pred_maxvals)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# c = meta['center'].numpy()
# s = meta['scale'].numpy()
# score = meta['score'].numpy()
preds, maxvals = get_final_preds(
config, output.clone().cpu().numpy(), None, None)
all_preds[idx:idx + num_images, :, 0:2] = preds[:, :, 0:2] * 4 # to go from hm size 64 to image size 256
all_preds[idx:idx + num_images, :, 2:3] = maxvals
# double check this all_boxes parts
# all_boxes[idx:idx + num_images, 0:2] = c[:, 0:2]
# all_boxes[idx:idx + num_images, 2:4] = s[:, 0:2]
# all_boxes[idx:idx + num_images, 4] = np.prod(s*200, 1)
# all_boxes[idx:idx + num_images, 5] = score
image_path.extend(meta['image'])
#Export annotations
for j in range(num_images):
annot = {"joints_vis": maxvals[j].squeeze().tolist(),
"joints": (pred[j]*4).tolist(),
"image": meta['image'][j]
}
export_annots.append(annot)
idx += num_images
if i % config.PRINT_FREQ == 0:
msg = 'Test: [{0}/{1}]\t' \
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' \
'Loss {loss.val:.4f} ({loss.avg:.4f})\t' \
'Accuracy {acc.val:.3f} ({acc.avg:.3f})'.format(
i, len(val_loader), batch_time=batch_time,
loss=losses, acc=acc)
logger.info(msg)
prefix = '{}_{}'.format(
os.path.join(output_dir, 'val'), i
)
save_debug_images(config, input, meta, target, pred*4, output,
prefix)
if config.LOCAL and i>10:
break
name_values, perf_indicator = val_dataset.evaluate(
config, all_preds, output_dir, all_boxes, image_path,
filenames, imgnums
)
model_name = config.MODEL.NAME
if isinstance(name_values, list):
for name_value in name_values:
_print_name_value_column(name_value, model_name)
else:
_print_name_value_column(name_values, model_name)
#Compute and display accuracy, precision and recall
target_weights = torch.cat(target_weights, dim=0).squeeze()
gt_vis = ~target_weights.cpu().numpy().astype(bool)
pred_max_vals_valid = np.concatenate(pred_max_vals_valid, axis=0)
msg_notvis = metrics_notvisible(pred_max_vals_valid, gt_vis)
logger.info(msg_notvis)
if writer_dict:
writer = writer_dict['writer']
global_steps = writer_dict['valid_global_steps']
writer.add_scalar(
'valid_loss',
losses.avg,
global_steps
)
writer.add_scalar(
'valid_acc',
acc.avg,
global_steps
)
if isinstance(name_values, list):
for name_value in name_values:
writer.add_scalars(
'valid',
dict(name_value),
global_steps
)
else:
writer.add_scalars(
'valid',
dict(name_values),
global_steps
)
writer_dict['valid_global_steps'] = global_steps + 1
with open(os.path.join(output_dir, '{}_pred_annots_{}.json'.format(config.DATASET.TEST_SET, time.strftime('%Y-%m-%d-%H-%M'))), 'w', encoding='utf-8') as f:
json.dump(export_annots, f, ensure_ascii=False, indent=4)
return perf_indicator
def inference(config, image_loader, image_dataset, model, output_dir):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
num_samples = len(image_dataset)
all_preds = np.zeros((num_samples, config.MODEL.NUM_JOINTS, 3),
dtype=np.float32)
all_boxes = np.zeros((num_samples, 5))
all_image_pathes = []
all_image_ids = []
idx = 0
with torch.no_grad():
end = time.time()
for i, (input, target, target_weight, meta) in enumerate(image_loader):
num_images = input.size(0)
# compute output
outputs = model(input)
if isinstance(outputs, list):
output = outputs[-1]
else:
output = outputs
if config.TEST.FLIP_TEST:
# this part is ugly, because pytorch has not supported negative index
# input_flipped = model(input[:, :, :, ::-1])
input_flipped = np.flip(input.cpu().numpy(), 3).copy()
input_flipped = torch.from_numpy(input_flipped).cuda()
outputs_flipped = model(input_flipped)
if isinstance(outputs_flipped, list):
output_flipped = outputs_flipped[-1]
else:
output_flipped = outputs_flipped
output_flipped = flip_back(output_flipped.cpu().numpy(),
image_dataset.flip_pairs)
output_flipped = torch.from_numpy(output_flipped.copy()).cuda()
# feature is not aligned, shift flipped heatmap for higher accuracy
if config.TEST.SHIFT_HEATMAP:
output_flipped[:, :, :, 1:] = \
output_flipped.clone()[:, :, :, 0:-1]
# output_flipped[:, :, :, 0] = 0
output = (output + output_flipped) * 0.5
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
c = meta['center'].numpy()
s = meta['scale'].numpy()
score = meta['score'].numpy()
tlwhs = meta['bbox_tlwh'].numpy()
output = output.data.cpu()
preds, maxvals = get_final_preds(config, output.numpy(), c, s)
all_preds[idx:idx + num_images, :, 0:2] = preds[:, :, 0:2]
all_preds[idx:idx + num_images, :, 2:3] = maxvals
# double check this all_boxes parts
all_boxes[idx:idx + num_images, 0:4] = tlwhs
all_boxes[idx:idx + num_images, 4] = score
all_image_pathes.extend(meta['image'])
if config.DATASET.DATASET == 'mot':
seq_names, frame_ids = meta['image_id']
frame_ids = frame_ids.numpy().astype(int)
all_image_ids.extend(list(zip(seq_names, frame_ids)))
elif config.DATASET.DATASET == 'aifi':
all_image_ids.extend(meta['image_id'])
idx += num_images
if i % config.PRINT_FREQ == 0:
msg = 'Test: [{0}/{1}]\t' \
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'.format(
i, len(image_loader), batch_time=batch_time)
logger.info(msg)
prefix = '{}_{}'.format(os.path.join(output_dir, 'inference'),
i)
pred, _ = get_max_preds(output.numpy())
save_debug_images(config, input, meta, target, pred * 4,
output, prefix)
# write output
frame_results = defaultdict(list)
for image_id, pred, box in zip(all_image_ids, all_preds, all_boxes):
frame_results[image_id].append(
(pred.astype(float).tolist(), box.astype(float).tolist()))
final_results = {}
for image_id, results in frame_results.items():
keypoints, boxes = zip(*results)
final_results[image_id] = {'keypoints': keypoints, 'boxes': boxes}
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with open(os.path.join(output_dir, 'box_keypoints.json'), 'w') as f:
json.dump(final_results, f)
logger.info('Save results to {}'.format(
os.path.join(output_dir, 'box_keypoints.json')))
def validate(config,
val_loader,
val_dataset,
model,
criterion,
output_dir,
tb_log_dir,
writer_dict=None):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
model.eval()
total_error = 0
with torch.no_grad():
end = time.time()
for i, (input, target, target_weight,
meta) in enumerate(val_loader): # compute output
output = model(input)
output_256 = model(meta['img_resize256_BN'])
if config.TEST.FLIP_TEST:
# this part is ugly, because pytorch has not supported negative index
# input_flipped = model(input[:, :, :, ::-1])
input_flipped = np.flip(input.cpu().numpy(),
3).copy() # 计算翻转图像的预测结果
input_flipped = torch.from_numpy(input_flipped).cuda()
output_flipped = model(input_flipped)
output_flipped = flip_back(output_flipped.cpu().numpy(),
val_dataset.flip_pairs)
output_flipped = torch.from_numpy(output_flipped.copy()).cuda()
# feature is not aligned, shift flipped heatmap for higher accuracy
if config.TEST.SHIFT_HEATMAP:
output_flipped[:, :, :, 1:] = output_flipped.clone(
)[:, :, :, 0:-1] # 将结果向右偏移1个单位
# output_flipped[:, :, :, 0] = 0
output = (output + output_flipped) * 0.5
target = target.cuda(non_blocking=True)
target_weight = target_weight.cuda(non_blocking=True)
loss = criterion(output, target, target_weight)
# 将heatmap的点，变成特征点上的图。
num_images = input.size(0)
losses.update(loss.item(), num_images)
_, avg_acc, cnt, pred = accuracy(output.cpu().numpy(),
target.cpu().numpy())
acc.update(avg_acc, cnt)
batch_time.update(time.time() - end)
end = time.time()
c = meta['center'].numpy(
) # 因为图像从Dataset里面出来的时候，加了个随机偏移，所以在测试的时候，要把图像偏回去
s = meta['scale'].numpy()
# 比较pred和gt的heatmap值
pred_heatmap, _ = get_max_preds(
output.clone().cpu().numpy()) # 预测值
target_heatmap, _ = get_max_preds(target.clone().cpu().numpy())
pred_heatmap_256, _ = get_max_preds(
output_256.clone().cpu().numpy())
pred_heatmap_256 *= 4
target_256_64, _ = get_max_preds(
meta['target_256_64'].clone().cpu().numpy())
target_256_64 *= 4
preds, maxvals = get_final_preds(config,
output.clone().cpu().numpy(), c,
s) # 变回到250那个尺度上
gt_landmark = meta['joints'].numpy()
imgs_256 = meta["img_256"]
# for img_idx in range(imgs_256.shape[0]):
# vis_face(imgs_256[img_idx], gt_landmark[img_idx], str(img_idx) + ".jpg")
# vis_face(imgs_256[img_idx], preds[img_idx], str(img_idx) + ".jpg")
# vis_face(meta['img_resize256'][img_idx], meta['joints_256'][img_idx], str(img_idx) + ".jpg")
# vis_face(meta['img_resize256'][img_idx], target_256_64[img_idx], "target_256_64"+str(img_idx) + ".jpg", show = False)
# vis_face(meta['img_resize256'][img_idx], pred_heatmap_256[img_idx], "pred_heatmap_256"+ str(img_idx) + ".jpg", show = False)
# batch_error_mean = normalisedError(gt_landmark, preds)
batch_error_mean = normalisedError(target_256_64, pred_heatmap_256)
total_error += batch_error_mean
total_mean_error = total_error / (i + 1)
print(
"batch id:{0}, current batch mean error is:{1}, total mean error is:{2}"
.format(i, batch_error_mean, total_mean_error))
def save_batch_heatmaps(batch_image, batch_heatmaps, file_name,
normalize=True):
'''
batch_image: [batch_size, channel, height, width]
batch_heatmaps: ['batch_size, num_joints, height, width]
file_name: saved file name
'''
if normalize:
batch_image = batch_image.clone()
min = float(batch_image.min())
max = float(batch_image.max())
batch_image.add_(-min).div_(max - min + 1e-5)
batch_size = batch_heatmaps.size(0)
num_joints = batch_heatmaps.size(1)
heatmap_height = batch_heatmaps.size(2)
heatmap_width = batch_heatmaps.size(3)
grid_image = np.zeros((batch_size*heatmap_height,
(num_joints+1)*heatmap_width,
3),
dtype=np.uint8)
preds, maxvals = get_max_preds(batch_heatmaps.detach().cpu().numpy())
##---new code---##
Rx = batch_image.size(3)/heatmap_width
Ry = batch_image.size(2)/heatmap_height
##--ends here--##
results = []
for i in range(batch_size):
image = batch_image[i].mul(255)\
.clamp(0, 255)\
.byte()\
.permute(1, 2, 0)\
.cpu().numpy()
heatmaps = batch_heatmaps[i].mul(255)\
.clamp(0, 255)\
.byte()\
.cpu().numpy()
resized_image = cv2.resize(image,
(int(heatmap_width), int(heatmap_height)))
height_begin = heatmap_height * i
height_end = heatmap_height * (i + 1)
for j in range(num_joints):
##--operations on resized image--##
cv2.circle(resized_image,
(int(preds[i][j][0]), int(preds[i][j][1])),
1, [0, 0, 255], 1)
heatmap = heatmaps[j, :, :]
colored_heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
masked_image = colored_heatmap*0.7 + resized_image*0.3
cv2.circle(masked_image,
(int(preds[i][j][0]), int(preds[i][j][1])),
1, [0, 0, 255], 1)
##--finishes here ig??--##
##--lets try new ops, on og image--##
cv2.circle(image,
(int(preds[i][j][0]*Rx), int(preds[i][j][1]*Ry)),
1, [0, 0, 255], 1)
heatmap_r = cv2.resize(heatmaps[j, :, :], (batch_image.size(3),batch_image.size(2)))
colored_heatmap_r = cv2.applyColorMap(heatmap_r, cv2.COLORMAP_JET)
masked_image_r = colored_heatmap_r*0.7 + image*0.3
cv2.circle(masked_image_r,
(int(preds[i][j][0]*Rx), int(preds[i][j][1]*Ry)),
3, [0, 0, 255], 1)
results.append(masked_image_r)
#cv2.imwrite(file_name[:9]+str(j)+file_name[9:], masked_image_r)
##--ripp--##
width_begin = heatmap_width * (j+1)
width_end = heatmap_width * (j+2)
grid_image[height_begin:height_end, width_begin:width_end, :] = \
masked_image
# grid_image[height_begin:height_end, width_begin:width_end, :] = \
# colored_heatmap*0.7 + resized_image*0.3
grid_image[height_begin:height_end, 0:heatmap_width, :] = resized_image
cv2.imwrite(file_name, grid_image)
return results
def main():
###mon = {'top': 160, 'left': 160, 'width': 200, 'height': 200}
###sct = mss()
# Specify the configuration file and model file for the neural network.
update_config("experiments/coco/resnet50/256x192_d256x3_adam_lr1e-3.yaml")
config.TEST.MODEL_FILE = "models/pytorch/pose_coco/pose_resnet_50_256x192.pth.tar"
###cap = cv2.VideoCapture(-1)
cap = cv2.VideoCapture('vid4.mp4') # Load the video
# Configure some settings for CUDA.
cudnn.benchmark = config.CUDNN.BENCHMARK
torch.backends.cudnn.deterministic = config.CUDNN.DETERMINISTIC
torch.backends.cudnn.enabled = config.CUDNN.ENABLED
model = eval('models.' + config.MODEL.NAME + '.get_pose_net')(
config, is_train=False)
# Load the model for the neural network.
model.load_state_dict(torch.load(config.TEST.MODEL_FILE))
gpus = [int(i) for i in config.GPUS.split(',')]
model = torch.nn.DataParallel(model, device_ids=gpus).cuda()
# Switch to evaluate mode (not training mode)
model.eval()
with torch.no_grad():
while True:
###img = sct.grab(mon)
###cap = Image.frombytes('RGB', (sct.width, sct.height), sct.image)
ret, data_numpy = cap.read() # Read an image from the frame.
# data_numpy.namedWindom("frame" , 0)
# cv2.resizeWindow("frame", 800,600);
# data_numpy = cv2.imread(frame, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
###model_input = cv2.resize(data_numpy, (192, 256))
model_input = cv2.resize(data_numpy, (192, 256),
fx=0,
fy=0,
interpolation=cv2.INTER_NEAREST)
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
model_input = transform(model_input).unsqueeze(0)
# Compute output heatmap
output = model(model_input)
# Compute coordinates of the joints.
output_matrix = output.clone().cpu().numpy()
preds, maxvals = get_max_preds(output_matrix)
# Display the points on the screen (rescaled to the original image dimensions).
image = data_numpy.copy()
y_scale, x_scale = image.shape[0] / output_matrix.shape[
2], image.shape[1] / output_matrix.shape[3]
keyp = []
right_shoulder = preds[0][6]
x, y = int(right_shoulder[0] * x_scale), int(right_shoulder[1] *
y_scale)
cv2.circle(image, (x, y), 2, (255, 0, 0), 2)
keyp.append(right_shoulder)
left_shoulder = preds[0][5]
x, y = int(left_shoulder[0] * x_scale), int(left_shoulder[1] *
y_scale)
cv2.circle(image, (x, y), 2, (255, 0, 0), 2)
keyp.append(left_shoulder)
left_hip = preds[0][11]
x, y = int(left_hip[0] * x_scale), int(left_hip[1] * y_scale)
cv2.circle(image, (x, y), 2, (255, 0, 0), 2)
keyp.append(left_hip)
right_hip = preds[0][12]
x, y = int(right_hip[0] * x_scale), int(right_hip[1] * y_scale)
cv2.circle(image, (x, y), 2, (255, 0, 0), 2)
keyp.append(right_hip)
center = (keyp[0] + keyp[1] + 1.5 * (keyp[2] + keyp[3]))
x, y = int(center[0] / 5 * x_scale), int(center[1] / 5 * y_scale)
cv2.circle(image, (x, y), 5, (0, 255, 0), 5)
cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS
cv2.imshow('Visualisation', image)
if cv2.waitKey(100) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
