def get_model_blob(input_image, params, model, model_params):
multiplier = [
x * model_params['boxsize'] / input_image.shape[0]
for x in params['scale_search']
]
blob = {"input_image_shape": input_image.shape, "model": {}}
for scale in multiplier:
image_to_test = cv2.resize(input_image, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
image_to_test_padded, pad = util.pad_right_down_corner(
image_to_test, model_params['stride'], model_params['padValue'])
# required shape (1, width, height, channels)
input_img = np.transpose(
np.float32(image_to_test_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2))
blob['model'][scale] = {
'image_to_test_padded_shape': image_to_test_padded.shape,
'pad': pad
}
blob['model'][scale]['output_blobs'] = model.predict(input_img)
return blob
def _extract_heat_map_and_paf(self, image):
"""
:param image: target image
:return: 18 layers heat map for body parts and 1 layer for background, paf,
detail to see OpenPose, https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md
"""
height, width = self.im_height, self.im_width
multiplier = [
x * model_params['boxsize'] / height
for x in params['scale_search']
]
multiplier_len = len(multiplier)
heat_map_average = np.zeros((height, width, 19))
paf_average = np.zeros((height, width, 38))
for scale in multiplier:
image2test = resize(image, fx=scale, fy=scale)
padded_image, pad = pad_right_down_corner(image2test,
model_params['stride'],
model_params['padValue'])
input_img = np.transpose(
np.float32(padded_image[:, :, :, np.newaxis]), (3, 0, 1, 2))
results = self.model.predict(input_img)
heat_map = np.squeeze(results[1])
heat_map = resize(heat_map,
fx=model_params['stride'],
fy=model_params['stride'])
heat_map = heat_map[:padded_image.shape[0] -
pad[2], :padded_image.shape[1] - pad[3], :]
heat_map = resize(heat_map, output_size=(width, height))
heat_map_average = heat_map_average + heat_map / multiplier_len
paf = np.squeeze(results[0]) # output 0 is PAFs
paf = resize(paf,
fx=model_params['stride'],
fy=model_params['stride'])
paf = paf[:padded_image.shape[0] - pad[2], :padded_image.shape[1] -
pad[3], :]
paf = resize(paf, output_size=(width, height))
paf_average = paf_average + paf / multiplier_len
return heat_map_average, paf_average
def shared_points(self, model, input_image):
from config_reader import config_reader
param, model_params = config_reader()
oriImg = input_image
self.oriImg = oriImg
multiplier = [x * model_params['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.pad_right_down_corner(imageToTest, model_params['stride'], model_params['padValue'])
input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2)) # required shape (1, width, height, channels)
output_blobs = model.predict(input_img)
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
from numpy import ma
U = paf_avg[:,:,16] * -1
V = paf_avg[:,:,17]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M)
from scipy.ndimage.filters import gaussian_filter
all_peaks = []
peak_counter = 0
for part in range(19-1):
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:]
peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
self.paf_avg = paf_avg
return all_peaks
def extract_parts_angle(input_image, params, model, model_params):
multiplier = [
x * model_params['boxsize'] / input_image.shape[0]
for x in params['scale_search']
]
# Body parts location heatmap, one per part (19)
heatmap_avg = np.zeros((input_image.shape[0], input_image.shape[1], 19))
# Part affinities, one per limb (38)
paf_avg = np.zeros((input_image.shape[0], input_image.shape[1], 38))
for scale in multiplier:
image_to_test = cv2.resize(input_image, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
image_to_test_padded, pad = util.pad_right_down_corner(
image_to_test, model_params['stride'], model_params['padValue'])
# required shape (1, width, height, channels)
input_img = np.transpose(
np.float32(image_to_test_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2))
output_blobs = model.predict(input_img)
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:image_to_test_padded.shape[0] -
pad[2], :image_to_test_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap,
(input_image.shape[1], input_image.shape[0]),
interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:image_to_test_padded.shape[0] -
pad[2], :image_to_test_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (input_image.shape[1], input_image.shape[0]),
interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
all_peaks = []
peak_counter = 0
for part in range(18):
hmap_ori = heatmap_avg[:, :, part]
hmap = gaussian_filter(hmap_ori, sigma=3)
# Find the pixel that has maximum value compared to those around it
hmap_left = np.zeros(hmap.shape)
hmap_left[1:, :] = hmap[:-1, :]
hmap_right = np.zeros(hmap.shape)
hmap_right[:-1, :] = hmap[1:, :]
hmap_up = np.zeros(hmap.shape)
hmap_up[:, 1:] = hmap[:, :-1]
hmap_down = np.zeros(hmap.shape)
hmap_down[:, :-1] = hmap[:, 1:]
# reduce needed because there are > 2 arguments
peaks_binary = np.logical_and.reduce(
(hmap >= hmap_left, hmap >= hmap_right, hmap >= hmap_up,
hmap >= hmap_down, hmap > params['thre1']))
peaks = list(
zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (hmap_ori[x[1], x[0]], ) for x in peaks
] # add a third element to tuple with score
idx = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (idx[i], ) for i in range(len(idx))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(util.hmapIdx)):
score_mid = paf_avg[:, :, [x - 19 for x in util.hmapIdx[k]]]
cand_a = all_peaks[util.limbSeq[k][0] - 1]
cand_b = all_peaks[util.limbSeq[k][1] - 1]
n_a = len(cand_a)
n_b = len(cand_b)
# index_a, index_b = util.limbSeq[k]
if n_a != 0 and n_b != 0:
connection_candidate = []
for i in range(n_a):
for j in range(n_b):
vec = np.subtract(cand_b[j][:2], cand_a[i][:2])
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
# failure case when 2 body parts overlaps
if norm == 0:
continue
vec = np.divide(vec, norm)
startend = list(
zip(
np.linspace(cand_a[i][0],
cand_b[j][0],
num=mid_num),
np.linspace(cand_a[i][1],
cand_b[j][1],
num=mid_num)))
vec_x = np.array([
score_mid[int(round(startend[I][1])),
int(round(startend[I][0])), 0]
for I in range(len(startend))
])
vec_y = np.array([
score_mid[int(round(startend[I][1])),
int(round(startend[I][0])), 1]
for I in range(len(startend))
])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
score_with_dist_prior = sum(score_midpts) / len(
score_midpts) + min(
0.5 * input_image.shape[0] / norm - 1, 0)
criterion1 = len(
np.nonzero(score_midpts > params['thre2'])
[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + cand_a[i][2] + cand_b[j][2]
])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if i not in connection[:, 3] and j not in connection[:, 4]:
connection = np.vstack(
[connection, [cand_a[i][3], cand_b[j][3], s, i, j]])
if len(connection) >= min(n_a, n_b):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = np.empty((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(util.hmapIdx)):
if k not in special_k:
part_as = connection_all[k][:, 0]
part_bs = connection_all[k][:, 1]
index_a, index_b = np.array(util.limbSeq[k]) - 1
for i in range(len(connection_all[k])): # = 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): # 1:size(subset,1):
if subset[j][index_a] == part_as[i] or subset[j][
index_b] == part_bs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if subset[j][index_b] != part_bs[i]:
subset[j][index_b] = part_bs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[part_bs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: # merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][index_b] = part_bs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
part_bs[i].astype(int),
2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[index_a] = part_as[i]
row[index_b] = part_bs[i]
row[-1] = 2
row[-2] = sum(
candidate[connection_all[k][i, :2].astype(int),
2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
delete_idx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
delete_idx.append(i)
subset = np.delete(subset, delete_idx, axis=0)
return all_peaks, subset, candidate
def getHeatMap_Avg(model, oriImg, model_params, multiplier, log=False):
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
if log == True:
# first figure shows padded images
f, axarr = plt.subplots(1, len(multiplier))
f.set_size_inches((20, 5))
# second figure shows heatmaps
f2, axarr2 = plt.subplots(1, len(multiplier))
f2.set_size_inches((20, 5))
# third figure shows PAFs
f3, axarr3 = plt.subplots(2, len(multiplier))
f3.set_size_inches((20, 10))
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.pad_right_down_corner(
imageToTest, model_params['stride'], model_params['padValue'])
if log == True:
axarr[m].imshow(imageToTest_padded[:, :, [2, 1, 0]])
axarr[m].set_title('Input image: scale %d' % m)
input_img = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2)) # required shape (1, width, height, channels)
print("Input shape: " + str(input_img.shape))
output_blobs = model.predict(input_img)
print("Output shape (heatmap): " + str(output_blobs[1].shape))
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
# visualization
if log == True:
axarr2[m].imshow(oriImg[:, :, [2, 1, 0]])
ax2 = axarr2[m].imshow(heatmap[:, :, 3], alpha=.5) # right elbow
axarr2[m].set_title('Heatmaps (Relb): scale %d' % m)
axarr3.flat[m].imshow(oriImg[:, :, [2, 1, 0]])
ax3x = axarr3.flat[m].imshow(paf[:, :, 16],
alpha=.5) # right elbow
axarr3.flat[m].set_title(
'PAFs (x comp. of Rwri to Relb): scale %d' % m)
axarr3.flat[len(multiplier) + m].imshow(oriImg[:, :, [2, 1, 0]])
ax3y = axarr3.flat[len(multiplier) + m].imshow(
paf[:, :, 17], alpha=.5) # right wrist
axarr3.flat[len(multiplier) + m].set_title(
'PAFs (y comp. of Relb to Rwri): scale %d' % m)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
if log == True:
f2.subplots_adjust(right=0.93)
cbar_ax = f2.add_axes([0.95, 0.15, 0.01, 0.7])
_ = f2.colorbar(ax2, cax=cbar_ax)
f3.subplots_adjust(right=0.93)
cbar_axx = f3.add_axes([0.95, 0.57, 0.01, 0.3])
_ = f3.colorbar(ax3x, cax=cbar_axx)
cbar_axy = f3.add_axes([0.95, 0.15, 0.01, 0.3])
_ = f3.colorbar(ax3y, cax=cbar_axy)
return heatmap_avg, paf_avg
def partCoordsOfImage(input_image):
oriImg = cv2.imread(input_image) # B,G,R order
#Load Configuration
param, model_params = config_reader()
multiplier = [
x * model_params['boxsize'] / oriImg.shape[0]
for x in param['scale_search']
]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
#Show sample heatmaps for right elbow and paf for right wrist and right elbow
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.pad_right_down_corner(
imageToTest, model_params['stride'], model_params['padValue'])
#axarr[m].imshow(imageToTest_padded[:,:,[2,1,0]])
#axarr[m].set_title('Input image: scale %d' % m)
input_img = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2)) # required shape (1, width, height, channels)
#print("Input shape: " + str(input_img.shape))
output_blobs = model.predict(input_img)
#print("Output shape (heatmap): " + str(output_blobs[1].shape))
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
#Visualise all detected body parts. Note that we use peaks in heatmaps
all_peaks = []
peak_counter = 0
for part in range(19 - 1):
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down,
map > param['thre1']))
peaks = list(
zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
[10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
[1,16], [16,18], [3,17], [6,18]]
# the middle joints heatmap correpondence
mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
[23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
[55,56], [37,38], [45,46]]
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]
candB = all_peaks[limbSeq[k][1] - 1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if (nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
# failure case when 2 body parts overlaps
if norm == 0:
continue
vec = np.divide(vec, norm)
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
score_with_dist_prior = sum(
score_midpts) / len(score_midpts) + min(
0.5 * oriImg.shape[0] / norm - 1, 0)
criterion1 = len(
np.nonzero(score_midpts > param['thre2'])
[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2] + candB[j][2]
])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if (i not in connection[:, 3] and j not in connection[:, 4]):
connection = np.vstack(
[connection, [candA[i][3], candB[j][3], s, i, j]])
if (len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][
indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if (subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
#print ("found = 2")
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(
candidate[connection_all[k][i, :2].astype(int),
2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
bestScore = 0
savedIndex = None
for x in range(len(subset)):
score = subset[x][-2]
if score > bestScore:
bestScore = score
savedIndex = x
bestPersonSubset = subset[savedIndex][:-2]
#print(bestScore)
#print(bestPersonSubset)
len(bestPersonSubset.astype(int))
partCoords = np.zeros((18, 2))
partCoords.fill(None)
for i in range(len(bestPersonSubset.astype(int))):
if bestPersonSubset[i] == -1.:
#print("-1 found on index", i)
continue
else:
partCoords[i][0] = list(chain.from_iterable(all_peaks))[
bestPersonSubset.astype(int)[i]][0]
partCoords[i][1] = list(chain.from_iterable(all_peaks))[
bestPersonSubset.astype(int)[i]][1]
return partCoords
def extract_parts(input_image, params, model, model_params):
"""
This function uses a Neural Network model to recognise persons and their body parts inside an image
:param input_image: cv2 image to analyse
:param params: parameters of the algorithm read from the config file
:param model: keras weights
:param model_params: parameters of the model read from the config file
:return:
an ndarray where each line is a subset (ideally a person), the first 18 columns are the ids of the parts which
map to the corresponding line in the second retruned variable, the 19th is the confidence of the subset and
the last one is the number of valid parts for that subset
an ndarray where each line is a part, the first two columns the x and y of the part, the third column the
confidence of that part and the fourth the part type
"""
multiplier = [
x * model_params['boxsize'] / input_image.shape[0]
for x in params['scale_search']
]
# Body parts location heatmap, one per part (19)
heatmap_avg = np.zeros((input_image.shape[0], input_image.shape[1], 19))
# Part affinities, one per limb (38)
paf_avg = np.zeros((input_image.shape[0], input_image.shape[1], 38))
for scale in multiplier:
image_to_test = cv2.resize(input_image, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
image_to_test_padded, pad = util.pad_right_down_corner(
image_to_test, model_params['stride'], model_params['padValue'])
# required shape (1, width, height, channels)
input_img = np.transpose(
np.float32(image_to_test_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2))
output_blobs = model.predict(input_img)
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:image_to_test_padded.shape[0] -
pad[2], :image_to_test_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap,
(input_image.shape[1], input_image.shape[0]),
interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:image_to_test_padded.shape[0] -
pad[2], :image_to_test_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (input_image.shape[1], input_image.shape[0]),
interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
# all_peaks is an array where each element contains an array of tuples. Each element of the tuple is a peak for
# a given body part
all_peaks = []
peak_counter = 0
for i in range(18):
hmap_ori = heatmap_avg[:, :, i]
hmap = gaussian_filter(hmap_ori, sigma=3)
# Find the pixel that has maximum value compared to those around it
hmap_left = np.zeros(hmap.shape)
hmap_left[1:, :] = hmap[:-1, :]
hmap_right = np.zeros(hmap.shape)
hmap_right[:-1, :] = hmap[1:, :]
hmap_up = np.zeros(hmap.shape)
hmap_up[:, 1:] = hmap[:, :-1]
hmap_down = np.zeros(hmap.shape)
hmap_down[:, :-1] = hmap[:, 1:]
# reduce needed because there are > 2 arguments
peaks_binary = np.logical_and.reduce(
(hmap >= hmap_left, hmap >= hmap_right, hmap >= hmap_up,
hmap >= hmap_down, hmap > params['thre1']))
# peaks will contain a tuple for each peak in the image
peaks = list(
zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (hmap_ori[x[1], x[0]], ) for x in peaks
] # add a third element to tuples (the score)
idx = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (idx[i], ) for i in range(len(idx))
] # add id to tuples
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(util.limb_paf_idx)):
# PAF for the given limbs (i.e. connection between parts)
score_mid = paf_avg[:, :, [x - 19 for x in util.limb_paf_idx[k]]]
# get all peaks for the given parts
cand_a = all_peaks[util.part_seq[k][0] -
1] # each element is [x, y, score, unique_id]
cand_b = all_peaks[util.part_seq[k][1] -
1] # each element is [x, y, score, unique_id]
n_a = len(cand_a)
n_b = len(cand_b)
# index_a, index_b = util.part_seq[k]
if n_a != 0 and n_b != 0:
connection_candidate = []
for i in range(n_a):
for j in range(n_b):
vec = np.subtract(cand_b[j][:2], cand_a[i][:2])
norm = np.linalg.norm(vec)
# failure case when 2 body parts overlaps
if norm == 0:
continue
vec = np.divide(vec, norm)
startend = list(
zip(
np.linspace(cand_a[i][0],
cand_b[j][0],
num=mid_num),
np.linspace(cand_a[i][1],
cand_b[j][1],
num=mid_num)))
startend = np.array(startend).round().astype(int)
vec_x = np.array(
[score_mid[se_i[1], se_i[0], 0] for se_i in startend])
vec_y = np.array(
[score_mid[se_i[1], se_i[0], 1] for se_i in startend])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
score_with_dist_prior = sum(
score_midpts) / len(score_midpts) + min(
0.5 * input_image.shape[0] / norm - 1, 0
) # mean of the points + penalty if segment is too big
criterion1 = len(
np.nonzero(score_midpts > params['thre2'])
[0]) > 0.8 * len(
score_midpts
) # If more than 80% of midpoints have an high score
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append(
[i, j, score_with_dist_prior])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
# Will contain [seq_id_a, seq_id_b, score, ith_of_a, jth_of_b]
connections = np.zeros((0, 5))
for cc in connection_candidate:
i, j, s = cc
if i not in connections[:, 3] and j not in connections[:, 4]:
connections = np.vstack(
[connections, [cand_a[i][3], cand_b[j][3], s, i, j]])
if len(connections) >= min(n_a, n_b):
break
connection_all.append(connections)
else:
special_k.append(k)
connection_all.append([])
# Each subset is a person (or at least should be)
# The last element of each row is the total number of parts belonging to that person
# The second last element of each row is the score of the overall configuration
# Here you basically map each part to the person they belong to
subsets = np.empty((0, 20))
# Put together all rows of all_peaks and add the part type
candidates = np.array([
item[:-1] + (i, ) for i, sublist in enumerate(all_peaks)
for item in sublist
])
del all_peaks
for k in range(len(util.limb_paf_idx)):
if k in special_k:
continue
connections = connection_all[k]
part_as = connections[:, 0]
part_bs = connections[:, 1]
index_a, index_b = util.part_seq[k] - 1
# For each connection of the limb
for i in range(connections.shape[0]):
found = 0
subset_idx = [-1, -1]
for j in range(subsets.shape[0]):
if subsets[j][index_a] == part_as[i] or subsets[j][
index_b] == part_bs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if subsets[j][index_b] != part_bs[i]:
subsets[j][index_b] = part_bs[i]
subsets[j][-1] += 1
subsets[j][-2] += candidates[part_bs[i].astype(int),
2] + connections[i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
membership = ((subsets[j1] >= 0).astype(int) +
(subsets[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: # merge
subsets[j1][:-2] += (subsets[j2][:-2] + 1)
subsets[j1][-2:] += subsets[j2][-2:]
subsets[j1][-2] += connections[i][2]
subsets = np.delete(subsets, j2, 0)
else: # as like found == 1
subsets[j1][index_b] = part_bs[i]
subsets[j1][-1] += 1
subsets[j1][-2] += candidates[part_bs[i].astype(int),
2] + connections[i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = np.full(20, fill_value=-1)
row[index_a] = part_as[
i] # sequential unique id of part a across persons
row[index_b] = part_bs[
i] # sequential unique id of part b across persons
row[-1] = 2
row[-2] = sum(candidates[connections[i, :2].astype(int),
2]) + connections[i][2]
subsets = np.vstack([subsets, row])
# delete some rows of subset which has few parts occur
delete_idx = []
for i in range(len(subsets)):
# Too few parts or low average score
if subsets[i][-1] < 4 or subsets[i][-2] / subsets[i][-1] < 0.4:
delete_idx.append(i)
subsets = np.delete(subsets, delete_idx, axis=0)
return subsets, candidates
def process(input_image, params, model_params):
oriImg = cv2.imread(input_image) # B,G,R order
multiplier = [
x * model_params['boxsize'] / oriImg.shape[0]
for x in params['scale_search']
]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0, 0),
fx=scale,
fy=scale,
interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.pad_right_down_corner(
imageToTest, model_params['stride'], model_params['padValue'])
input_img = np.transpose(
np.float32(imageToTest_padded[:, :, :, np.newaxis]),
(3, 0, 1, 2)) # required shape (1, width, height, channels)
output_blobs = model.predict(input_img)
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0),
fx=model_params['stride'],
fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0] -
pad[2], :imageToTest_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]),
interpolation=cv2.INTER_CUBIC)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
all_peaks = []
peak_counter = 0
for part in range(18):
map_ori = heatmap_avg[:, :, part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down,
map > params['thre1']))
peaks = list(
zip(np.nonzero(peaks_binary)[1],
np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [
peaks_with_score[i] + (id[i], ) for i in range(len(id))
]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]
candB = all_peaks[limbSeq[k][1] - 1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if nA != 0 and nB != 0:
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
# failure case when 2 body parts overlaps
if norm == 0:
continue
vec = np.divide(vec, norm)
startend = list(
zip(np.linspace(candA[i][0], candB[j][0], num=mid_num),
np.linspace(candA[i][1], candB[j][1],
num=mid_num)))
vec_x = np.array(
[score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array(
[score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(
vec_y, vec[1])
score_with_dist_prior = sum(
score_midpts) / len(score_midpts) + min(
0.5 * oriImg.shape[0] / norm - 1, 0)
criterion1 = len(
np.nonzero(score_midpts > params['thre2'])
[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([
i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2] + candB[j][2]
])
connection_candidate = sorted(connection_candidate,
key=lambda x: x[2],
reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if i not in connection[:, 3] and j not in connection[:, 4]:
connection = np.vstack(
[connection, [candA[i][3], candB[j][3], s, i, j]])
if len(connection) >= min(nA, nB):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): # = 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): # 1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][
indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if subset[j][indexB] != partBs[i]:
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int),
2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
membership = ((subset[j1] >= 0).astype(int) +
(subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: # merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[
partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \
connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = []
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
canvas = cv2.imread(input_image) # B,G,R order
print('---------len subset: {}'.format(len(subset)))
persons_limbs = []
for n in range(len(subset)):
limbs = []
for i in range(17):
index = subset[n][np.array(limbSeq[i]) - 1]
if -1 in index:
limbs.append([])
continue
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
limbs.append([(X[0], Y[0]), (X[1], Y[1])])
# cv2.line(canvas, (int(Y[0]), int(X[0])), (int(Y[1]), int(X[1])), colors[n % 18], 2)
persons_limbs.append(limbs)
height, width = oriImg.shape[:2]
head_pose = HeadPoseEstimator((height, width), mode='nose_eyes_ears')
for limbs in persons_limbs:
body_pose = geo.BodyPose.from_connected_body_parts(limbs)
# geo.draw_body_pose_key_points(body_pose, canvas, colors)
# if body_pose.is_hands_up(threshold=30):
# print('----------hand up----------')
# x1, y1, x2, y2 = body_pose.mbr()
# cv2.rectangle(canvas, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)
# cv2.putText(canvas, 'Hand up', (int(x1), int(y1) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2)
# if body_pose.left_elbow is not None:
# print('===left_elbow===: {}'.format(body_pose.nose))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.left_elbow), 4, (0, 255, 255), thickness=-1)
#
# if body_pose.nose is not None:
# print('===nose===: {}'.format(body_pose.nose))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.nose), 4, (0, 255, 0), thickness=-1)
# if body_pose.left_wrist is not None:
# print('===left_wrist===: {}'.format(body_pose.left_wrist))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.left_wrist), 4, (255, 0, 0), thickness=-1)
# if body_pose.right_wrist is not None:
# print('===right_wrist===: {}'.format(body_pose.right_wrist))
# cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.right_wrist), 4, (0, 0, 255), thickness=-1)
body_pose.head_direction(canvas, offset2nose=30)
# print(body_pose.nose_2eyes())
# if body_pose.head_key_points() is not None:
# rotation_vector, translation_vector = head_pose.solve_pose(body_pose.head_key_points())
# print('r: {}, t: {}'.format(rotation_vector, translation_vector))
# points_2d = head_pose.projection(rotation_vector, translation_vector)
# points_2d = np.array(points_2d).astype(np.int).tolist()
# print('points: {}'.format(points_2d))
# geo.draw_head_pose(canvas, points_2d)
return canvas
