def get_model_blob(input_image, params, model, model_params): multiplier = [ x * model_params['boxsize'] / input_image.shape[0] for x in params['scale_search'] ] blob = {"input_image_shape": input_image.shape, "model": {}} for scale in multiplier: image_to_test = cv2.resize(input_image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) image_to_test_padded, pad = util.pad_right_down_corner( image_to_test, model_params['stride'], model_params['padValue']) # required shape (1, width, height, channels) input_img = np.transpose( np.float32(image_to_test_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) blob['model'][scale] = { 'image_to_test_padded_shape': image_to_test_padded.shape, 'pad': pad } blob['model'][scale]['output_blobs'] = model.predict(input_img) return blob
def _extract_heat_map_and_paf(self, image): """ :param image: target image :return: 18 layers heat map for body parts and 1 layer for background, paf, detail to see OpenPose, https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md """ height, width = self.im_height, self.im_width multiplier = [ x * model_params['boxsize'] / height for x in params['scale_search'] ] multiplier_len = len(multiplier) heat_map_average = np.zeros((height, width, 19)) paf_average = np.zeros((height, width, 38)) for scale in multiplier: image2test = resize(image, fx=scale, fy=scale) padded_image, pad = pad_right_down_corner(image2test, model_params['stride'], model_params['padValue']) input_img = np.transpose( np.float32(padded_image[:, :, :, np.newaxis]), (3, 0, 1, 2)) results = self.model.predict(input_img) heat_map = np.squeeze(results[1]) heat_map = resize(heat_map, fx=model_params['stride'], fy=model_params['stride']) heat_map = heat_map[:padded_image.shape[0] - pad[2], :padded_image.shape[1] - pad[3], :] heat_map = resize(heat_map, output_size=(width, height)) heat_map_average = heat_map_average + heat_map / multiplier_len paf = np.squeeze(results[0]) # output 0 is PAFs paf = resize(paf, fx=model_params['stride'], fy=model_params['stride']) paf = paf[:padded_image.shape[0] - pad[2], :padded_image.shape[1] - pad[3], :] paf = resize(paf, output_size=(width, height)) paf_average = paf_average + paf / multiplier_len return heat_map_average, paf_average
def shared_points(self, model, input_image): from config_reader import config_reader param, model_params = config_reader() oriImg = input_image self.oriImg = oriImg multiplier = [x * model_params['boxsize'] / oriImg.shape[0] for x in param['scale_search']] heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) for m in range(len(multiplier)): scale = multiplier[m] imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) imageToTest_padded, pad = util.pad_right_down_corner(imageToTest, model_params['stride'], model_params['padValue']) input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2)) # required shape (1, width, height, channels) output_blobs = model.predict(input_img) # extract outputs, resize, and remove padding heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :] heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :] paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier) paf_avg = paf_avg + paf / len(multiplier) from numpy import ma U = paf_avg[:,:,16] * -1 V = paf_avg[:,:,17] X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0])) M = np.zeros(U.shape, dtype='bool') M[U**2 + V**2 < 0.5 * 0.5] = True U = ma.masked_array(U, mask=M) V = ma.masked_array(V, mask=M) from scipy.ndimage.filters import gaussian_filter all_peaks = [] peak_counter = 0 for part in range(19-1): map_ori = heatmap_avg[:,:,part] map = gaussian_filter(map_ori, sigma=3) map_left = np.zeros(map.shape) map_left[1:,:] = map[:-1,:] map_right = np.zeros(map.shape) map_right[:-1,:] = map[1:,:] map_up = np.zeros(map.shape) map_up[:,1:] = map[:,:-1] map_down = np.zeros(map.shape) map_down[:,:-1] = map[:,1:] peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1'])) peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks] id = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))] all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) self.paf_avg = paf_avg return all_peaks
def extract_parts_angle(input_image, params, model, model_params): multiplier = [ x * model_params['boxsize'] / input_image.shape[0] for x in params['scale_search'] ] # Body parts location heatmap, one per part (19) heatmap_avg = np.zeros((input_image.shape[0], input_image.shape[1], 19)) # Part affinities, one per limb (38) paf_avg = np.zeros((input_image.shape[0], input_image.shape[1], 38)) for scale in multiplier: image_to_test = cv2.resize(input_image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) image_to_test_padded, pad = util.pad_right_down_corner( image_to_test, model_params['stride'], model_params['padValue']) # required shape (1, width, height, channels) input_img = np.transpose( np.float32(image_to_test_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) output_blobs = model.predict(input_img) # extract outputs, resize, and remove padding heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:image_to_test_padded.shape[0] - pad[2], :image_to_test_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:image_to_test_padded.shape[0] - pad[2], :image_to_test_padded.shape[1] - pad[3], :] paf = cv2.resize(paf, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier) paf_avg = paf_avg + paf / len(multiplier) all_peaks = [] peak_counter = 0 for part in range(18): hmap_ori = heatmap_avg[:, :, part] hmap = gaussian_filter(hmap_ori, sigma=3) # Find the pixel that has maximum value compared to those around it hmap_left = np.zeros(hmap.shape) hmap_left[1:, :] = hmap[:-1, :] hmap_right = np.zeros(hmap.shape) hmap_right[:-1, :] = hmap[1:, :] hmap_up = np.zeros(hmap.shape) hmap_up[:, 1:] = hmap[:, :-1] hmap_down = np.zeros(hmap.shape) hmap_down[:, :-1] = hmap[:, 1:] # reduce needed because there are > 2 arguments peaks_binary = np.logical_and.reduce( (hmap >= hmap_left, hmap >= hmap_right, hmap >= hmap_up, hmap >= hmap_down, hmap > params['thre1'])) peaks = list( zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (hmap_ori[x[1], x[0]], ) for x in peaks ] # add a third element to tuple with score idx = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [ peaks_with_score[i] + (idx[i], ) for i in range(len(idx)) ] all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) connection_all = [] special_k = [] mid_num = 10 for k in range(len(util.hmapIdx)): score_mid = paf_avg[:, :, [x - 19 for x in util.hmapIdx[k]]] cand_a = all_peaks[util.limbSeq[k][0] - 1] cand_b = all_peaks[util.limbSeq[k][1] - 1] n_a = len(cand_a) n_b = len(cand_b) # index_a, index_b = util.limbSeq[k] if n_a != 0 and n_b != 0: connection_candidate = [] for i in range(n_a): for j in range(n_b): vec = np.subtract(cand_b[j][:2], cand_a[i][:2]) norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) # failure case when 2 body parts overlaps if norm == 0: continue vec = np.divide(vec, norm) startend = list( zip( np.linspace(cand_a[i][0], cand_b[j][0], num=mid_num), np.linspace(cand_a[i][1], cand_b[j][1], num=mid_num))) vec_x = np.array([ score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] for I in range(len(startend)) ]) vec_y = np.array([ score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] for I in range(len(startend)) ]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply( vec_y, vec[1]) score_with_dist_prior = sum(score_midpts) / len( score_midpts) + min( 0.5 * input_image.shape[0] / norm - 1, 0) criterion1 = len( np.nonzero(score_midpts > params['thre2']) [0]) > 0.8 * len(score_midpts) criterion2 = score_with_dist_prior > 0 if criterion1 and criterion2: connection_candidate.append([ i, j, score_with_dist_prior, score_with_dist_prior + cand_a[i][2] + cand_b[j][2] ]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) connection = np.zeros((0, 5)) for c in range(len(connection_candidate)): i, j, s = connection_candidate[c][0:3] if i not in connection[:, 3] and j not in connection[:, 4]: connection = np.vstack( [connection, [cand_a[i][3], cand_b[j][3], s, i, j]]) if len(connection) >= min(n_a, n_b): break connection_all.append(connection) else: special_k.append(k) connection_all.append([]) # last number in each row is the total parts number of that person # the second last number in each row is the score of the overall configuration subset = np.empty((0, 20)) candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(util.hmapIdx)): if k not in special_k: part_as = connection_all[k][:, 0] part_bs = connection_all[k][:, 1] index_a, index_b = np.array(util.limbSeq[k]) - 1 for i in range(len(connection_all[k])): # = 1:size(temp,1) found = 0 subset_idx = [-1, -1] for j in range(len(subset)): # 1:size(subset,1): if subset[j][index_a] == part_as[i] or subset[j][ index_b] == part_bs[i]: subset_idx[found] = j found += 1 if found == 1: j = subset_idx[0] if subset[j][index_b] != part_bs[i]: subset[j][index_b] = part_bs[i] subset[j][-1] += 1 subset[j][-2] += candidate[part_bs[i].astype(int), 2] + connection_all[k][i][2] elif found == 2: # if found 2 and disjoint, merge them j1, j2 = subset_idx membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] if len(np.nonzero(membership == 2)[0]) == 0: # merge subset[j1][:-2] += (subset[j2][:-2] + 1) subset[j1][-2:] += subset[j2][-2:] subset[j1][-2] += connection_all[k][i][2] subset = np.delete(subset, j2, 0) else: # as like found == 1 subset[j1][index_b] = part_bs[i] subset[j1][-1] += 1 subset[j1][-2] += candidate[ part_bs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(20) row[index_a] = part_as[i] row[index_b] = part_bs[i] row[-1] = 2 row[-2] = sum( candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2] subset = np.vstack([subset, row]) # delete some rows of subset which has few parts occur delete_idx = [] for i in range(len(subset)): if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: delete_idx.append(i) subset = np.delete(subset, delete_idx, axis=0) return all_peaks, subset, candidate
def getHeatMap_Avg(model, oriImg, model_params, multiplier, log=False): heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) if log == True: # first figure shows padded images f, axarr = plt.subplots(1, len(multiplier)) f.set_size_inches((20, 5)) # second figure shows heatmaps f2, axarr2 = plt.subplots(1, len(multiplier)) f2.set_size_inches((20, 5)) # third figure shows PAFs f3, axarr3 = plt.subplots(2, len(multiplier)) f3.set_size_inches((20, 10)) for m in range(len(multiplier)): scale = multiplier[m] imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) imageToTest_padded, pad = util.pad_right_down_corner( imageToTest, model_params['stride'], model_params['padValue']) if log == True: axarr[m].imshow(imageToTest_padded[:, :, [2, 1, 0]]) axarr[m].set_title('Input image: scale %d' % m) input_img = np.transpose( np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) # required shape (1, width, height, channels) print("Input shape: " + str(input_img.shape)) output_blobs = model.predict(input_img) print("Output shape (heatmap): " + str(output_blobs[1].shape)) # extract outputs, resize, and remove padding heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) # visualization if log == True: axarr2[m].imshow(oriImg[:, :, [2, 1, 0]]) ax2 = axarr2[m].imshow(heatmap[:, :, 3], alpha=.5) # right elbow axarr2[m].set_title('Heatmaps (Relb): scale %d' % m) axarr3.flat[m].imshow(oriImg[:, :, [2, 1, 0]]) ax3x = axarr3.flat[m].imshow(paf[:, :, 16], alpha=.5) # right elbow axarr3.flat[m].set_title( 'PAFs (x comp. of Rwri to Relb): scale %d' % m) axarr3.flat[len(multiplier) + m].imshow(oriImg[:, :, [2, 1, 0]]) ax3y = axarr3.flat[len(multiplier) + m].imshow( paf[:, :, 17], alpha=.5) # right wrist axarr3.flat[len(multiplier) + m].set_title( 'PAFs (y comp. of Relb to Rwri): scale %d' % m) heatmap_avg = heatmap_avg + heatmap / len(multiplier) paf_avg = paf_avg + paf / len(multiplier) if log == True: f2.subplots_adjust(right=0.93) cbar_ax = f2.add_axes([0.95, 0.15, 0.01, 0.7]) _ = f2.colorbar(ax2, cax=cbar_ax) f3.subplots_adjust(right=0.93) cbar_axx = f3.add_axes([0.95, 0.57, 0.01, 0.3]) _ = f3.colorbar(ax3x, cax=cbar_axx) cbar_axy = f3.add_axes([0.95, 0.15, 0.01, 0.3]) _ = f3.colorbar(ax3y, cax=cbar_axy) return heatmap_avg, paf_avg
def partCoordsOfImage(input_image): oriImg = cv2.imread(input_image) # B,G,R order #Load Configuration param, model_params = config_reader() multiplier = [ x * model_params['boxsize'] / oriImg.shape[0] for x in param['scale_search'] ] heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) #Show sample heatmaps for right elbow and paf for right wrist and right elbow for m in range(len(multiplier)): scale = multiplier[m] imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) imageToTest_padded, pad = util.pad_right_down_corner( imageToTest, model_params['stride'], model_params['padValue']) #axarr[m].imshow(imageToTest_padded[:,:,[2,1,0]]) #axarr[m].set_title('Input image: scale %d' % m) input_img = np.transpose( np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) # required shape (1, width, height, channels) #print("Input shape: " + str(input_img.shape)) output_blobs = model.predict(input_img) #print("Output shape (heatmap): " + str(output_blobs[1].shape)) # extract outputs, resize, and remove padding heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier) paf_avg = paf_avg + paf / len(multiplier) #Visualise all detected body parts. Note that we use peaks in heatmaps all_peaks = [] peak_counter = 0 for part in range(19 - 1): map_ori = heatmap_avg[:, :, part] map = gaussian_filter(map_ori, sigma=3) map_left = np.zeros(map.shape) map_left[1:, :] = map[:-1, :] map_right = np.zeros(map.shape) map_right[:-1, :] = map[1:, :] map_up = np.zeros(map.shape) map_up[:, 1:] = map[:, :-1] map_down = np.zeros(map.shape) map_down[:, :-1] = map[:, 1:] peaks_binary = np.logical_and.reduce( (map >= map_left, map >= map_right, map >= map_up, map >= map_down, map > param['thre1'])) peaks = list( zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks] id = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [ peaks_with_score[i] + (id[i], ) for i in range(len(id)) ] all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) # find connection in the specified sequence, center 29 is in the position 15 limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \ [10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \ [1,16], [16,18], [3,17], [6,18]] # the middle joints heatmap correpondence mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \ [23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \ [55,56], [37,38], [45,46]] connection_all = [] special_k = [] mid_num = 10 for k in range(len(mapIdx)): score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]] candA = all_peaks[limbSeq[k][0] - 1] candB = all_peaks[limbSeq[k][1] - 1] nA = len(candA) nB = len(candB) indexA, indexB = limbSeq[k] if (nA != 0 and nB != 0): connection_candidate = [] for i in range(nA): for j in range(nB): vec = np.subtract(candB[j][:2], candA[i][:2]) norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) # failure case when 2 body parts overlaps if norm == 0: continue vec = np.divide(vec, norm) startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \ np.linspace(candA[i][1], candB[j][1], num=mid_num))) vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \ for I in range(len(startend))]) vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \ for I in range(len(startend))]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply( vec_y, vec[1]) score_with_dist_prior = sum( score_midpts) / len(score_midpts) + min( 0.5 * oriImg.shape[0] / norm - 1, 0) criterion1 = len( np.nonzero(score_midpts > param['thre2']) [0]) > 0.8 * len(score_midpts) criterion2 = score_with_dist_prior > 0 if criterion1 and criterion2: connection_candidate.append([ i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2] ]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) connection = np.zeros((0, 5)) for c in range(len(connection_candidate)): i, j, s = connection_candidate[c][0:3] if (i not in connection[:, 3] and j not in connection[:, 4]): connection = np.vstack( [connection, [candA[i][3], candB[j][3], s, i, j]]) if (len(connection) >= min(nA, nB)): break connection_all.append(connection) else: special_k.append(k) connection_all.append([]) # last number in each row is the total parts number of that person # the second last number in each row is the score of the overall configuration subset = -1 * np.ones((0, 20)) candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(mapIdx)): if k not in special_k: partAs = connection_all[k][:, 0] partBs = connection_all[k][:, 1] indexA, indexB = np.array(limbSeq[k]) - 1 for i in range(len(connection_all[k])): #= 1:size(temp,1) found = 0 subset_idx = [-1, -1] for j in range(len(subset)): #1:size(subset,1): if subset[j][indexA] == partAs[i] or subset[j][ indexB] == partBs[i]: subset_idx[found] = j found += 1 if found == 1: j = subset_idx[0] if (subset[j][indexB] != partBs[i]): subset[j][indexB] = partBs[i] subset[j][-1] += 1 subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] elif found == 2: # if found 2 and disjoint, merge them j1, j2 = subset_idx #print ("found = 2") membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] if len(np.nonzero(membership == 2)[0]) == 0: #merge subset[j1][:-2] += (subset[j2][:-2] + 1) subset[j1][-2:] += subset[j2][-2:] subset[j1][-2] += connection_all[k][i][2] subset = np.delete(subset, j2, 0) else: # as like found == 1 subset[j1][indexB] = partBs[i] subset[j1][-1] += 1 subset[j1][-2] += candidate[ partBs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(20) row[indexA] = partAs[i] row[indexB] = partBs[i] row[-1] = 2 row[-2] = sum( candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2] subset = np.vstack([subset, row]) # delete some rows of subset which has few parts occur deleteIdx = [] for i in range(len(subset)): if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: deleteIdx.append(i) subset = np.delete(subset, deleteIdx, axis=0) bestScore = 0 savedIndex = None for x in range(len(subset)): score = subset[x][-2] if score > bestScore: bestScore = score savedIndex = x bestPersonSubset = subset[savedIndex][:-2] #print(bestScore) #print(bestPersonSubset) len(bestPersonSubset.astype(int)) partCoords = np.zeros((18, 2)) partCoords.fill(None) for i in range(len(bestPersonSubset.astype(int))): if bestPersonSubset[i] == -1.: #print("-1 found on index", i) continue else: partCoords[i][0] = list(chain.from_iterable(all_peaks))[ bestPersonSubset.astype(int)[i]][0] partCoords[i][1] = list(chain.from_iterable(all_peaks))[ bestPersonSubset.astype(int)[i]][1] return partCoords
def extract_parts(input_image, params, model, model_params): """ This function uses a Neural Network model to recognise persons and their body parts inside an image :param input_image: cv2 image to analyse :param params: parameters of the algorithm read from the config file :param model: keras weights :param model_params: parameters of the model read from the config file :return: an ndarray where each line is a subset (ideally a person), the first 18 columns are the ids of the parts which map to the corresponding line in the second retruned variable, the 19th is the confidence of the subset and the last one is the number of valid parts for that subset an ndarray where each line is a part, the first two columns the x and y of the part, the third column the confidence of that part and the fourth the part type """ multiplier = [ x * model_params['boxsize'] / input_image.shape[0] for x in params['scale_search'] ] # Body parts location heatmap, one per part (19) heatmap_avg = np.zeros((input_image.shape[0], input_image.shape[1], 19)) # Part affinities, one per limb (38) paf_avg = np.zeros((input_image.shape[0], input_image.shape[1], 38)) for scale in multiplier: image_to_test = cv2.resize(input_image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) image_to_test_padded, pad = util.pad_right_down_corner( image_to_test, model_params['stride'], model_params['padValue']) # required shape (1, width, height, channels) input_img = np.transpose( np.float32(image_to_test_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) output_blobs = model.predict(input_img) # extract outputs, resize, and remove padding heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:image_to_test_padded.shape[0] - pad[2], :image_to_test_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:image_to_test_padded.shape[0] - pad[2], :image_to_test_padded.shape[1] - pad[3], :] paf = cv2.resize(paf, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier) paf_avg = paf_avg + paf / len(multiplier) # all_peaks is an array where each element contains an array of tuples. Each element of the tuple is a peak for # a given body part all_peaks = [] peak_counter = 0 for i in range(18): hmap_ori = heatmap_avg[:, :, i] hmap = gaussian_filter(hmap_ori, sigma=3) # Find the pixel that has maximum value compared to those around it hmap_left = np.zeros(hmap.shape) hmap_left[1:, :] = hmap[:-1, :] hmap_right = np.zeros(hmap.shape) hmap_right[:-1, :] = hmap[1:, :] hmap_up = np.zeros(hmap.shape) hmap_up[:, 1:] = hmap[:, :-1] hmap_down = np.zeros(hmap.shape) hmap_down[:, :-1] = hmap[:, 1:] # reduce needed because there are > 2 arguments peaks_binary = np.logical_and.reduce( (hmap >= hmap_left, hmap >= hmap_right, hmap >= hmap_up, hmap >= hmap_down, hmap > params['thre1'])) # peaks will contain a tuple for each peak in the image peaks = list( zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (hmap_ori[x[1], x[0]], ) for x in peaks ] # add a third element to tuples (the score) idx = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [ peaks_with_score[i] + (idx[i], ) for i in range(len(idx)) ] # add id to tuples all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) connection_all = [] special_k = [] mid_num = 10 for k in range(len(util.limb_paf_idx)): # PAF for the given limbs (i.e. connection between parts) score_mid = paf_avg[:, :, [x - 19 for x in util.limb_paf_idx[k]]] # get all peaks for the given parts cand_a = all_peaks[util.part_seq[k][0] - 1] # each element is [x, y, score, unique_id] cand_b = all_peaks[util.part_seq[k][1] - 1] # each element is [x, y, score, unique_id] n_a = len(cand_a) n_b = len(cand_b) # index_a, index_b = util.part_seq[k] if n_a != 0 and n_b != 0: connection_candidate = [] for i in range(n_a): for j in range(n_b): vec = np.subtract(cand_b[j][:2], cand_a[i][:2]) norm = np.linalg.norm(vec) # failure case when 2 body parts overlaps if norm == 0: continue vec = np.divide(vec, norm) startend = list( zip( np.linspace(cand_a[i][0], cand_b[j][0], num=mid_num), np.linspace(cand_a[i][1], cand_b[j][1], num=mid_num))) startend = np.array(startend).round().astype(int) vec_x = np.array( [score_mid[se_i[1], se_i[0], 0] for se_i in startend]) vec_y = np.array( [score_mid[se_i[1], se_i[0], 1] for se_i in startend]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply( vec_y, vec[1]) score_with_dist_prior = sum( score_midpts) / len(score_midpts) + min( 0.5 * input_image.shape[0] / norm - 1, 0 ) # mean of the points + penalty if segment is too big criterion1 = len( np.nonzero(score_midpts > params['thre2']) [0]) > 0.8 * len( score_midpts ) # If more than 80% of midpoints have an high score criterion2 = score_with_dist_prior > 0 if criterion1 and criterion2: connection_candidate.append( [i, j, score_with_dist_prior]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) # Will contain [seq_id_a, seq_id_b, score, ith_of_a, jth_of_b] connections = np.zeros((0, 5)) for cc in connection_candidate: i, j, s = cc if i not in connections[:, 3] and j not in connections[:, 4]: connections = np.vstack( [connections, [cand_a[i][3], cand_b[j][3], s, i, j]]) if len(connections) >= min(n_a, n_b): break connection_all.append(connections) else: special_k.append(k) connection_all.append([]) # Each subset is a person (or at least should be) # The last element of each row is the total number of parts belonging to that person # The second last element of each row is the score of the overall configuration # Here you basically map each part to the person they belong to subsets = np.empty((0, 20)) # Put together all rows of all_peaks and add the part type candidates = np.array([ item[:-1] + (i, ) for i, sublist in enumerate(all_peaks) for item in sublist ]) del all_peaks for k in range(len(util.limb_paf_idx)): if k in special_k: continue connections = connection_all[k] part_as = connections[:, 0] part_bs = connections[:, 1] index_a, index_b = util.part_seq[k] - 1 # For each connection of the limb for i in range(connections.shape[0]): found = 0 subset_idx = [-1, -1] for j in range(subsets.shape[0]): if subsets[j][index_a] == part_as[i] or subsets[j][ index_b] == part_bs[i]: subset_idx[found] = j found += 1 if found == 1: j = subset_idx[0] if subsets[j][index_b] != part_bs[i]: subsets[j][index_b] = part_bs[i] subsets[j][-1] += 1 subsets[j][-2] += candidates[part_bs[i].astype(int), 2] + connections[i][2] elif found == 2: # if found 2 and disjoint, merge them j1, j2 = subset_idx membership = ((subsets[j1] >= 0).astype(int) + (subsets[j2] >= 0).astype(int))[:-2] if len(np.nonzero(membership == 2)[0]) == 0: # merge subsets[j1][:-2] += (subsets[j2][:-2] + 1) subsets[j1][-2:] += subsets[j2][-2:] subsets[j1][-2] += connections[i][2] subsets = np.delete(subsets, j2, 0) else: # as like found == 1 subsets[j1][index_b] = part_bs[i] subsets[j1][-1] += 1 subsets[j1][-2] += candidates[part_bs[i].astype(int), 2] + connections[i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = np.full(20, fill_value=-1) row[index_a] = part_as[ i] # sequential unique id of part a across persons row[index_b] = part_bs[ i] # sequential unique id of part b across persons row[-1] = 2 row[-2] = sum(candidates[connections[i, :2].astype(int), 2]) + connections[i][2] subsets = np.vstack([subsets, row]) # delete some rows of subset which has few parts occur delete_idx = [] for i in range(len(subsets)): # Too few parts or low average score if subsets[i][-1] < 4 or subsets[i][-2] / subsets[i][-1] < 0.4: delete_idx.append(i) subsets = np.delete(subsets, delete_idx, axis=0) return subsets, candidates
def process(input_image, params, model_params): oriImg = cv2.imread(input_image) # B,G,R order multiplier = [ x * model_params['boxsize'] / oriImg.shape[0] for x in params['scale_search'] ] heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) for m in range(len(multiplier)): scale = multiplier[m] imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) imageToTest_padded, pad = util.pad_right_down_corner( imageToTest, model_params['stride'], model_params['padValue']) input_img = np.transpose( np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 0, 1, 2)) # required shape (1, width, height, channels) output_blobs = model.predict(input_img) # extract outputs, resize, and remove padding heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier) paf_avg = paf_avg + paf / len(multiplier) all_peaks = [] peak_counter = 0 for part in range(18): map_ori = heatmap_avg[:, :, part] map = gaussian_filter(map_ori, sigma=3) map_left = np.zeros(map.shape) map_left[1:, :] = map[:-1, :] map_right = np.zeros(map.shape) map_right[:-1, :] = map[1:, :] map_up = np.zeros(map.shape) map_up[:, 1:] = map[:, :-1] map_down = np.zeros(map.shape) map_down[:, :-1] = map[:, 1:] peaks_binary = np.logical_and.reduce( (map >= map_left, map >= map_right, map >= map_up, map >= map_down, map > params['thre1'])) peaks = list( zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (map_ori[x[1], x[0]], ) for x in peaks] id = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [ peaks_with_score[i] + (id[i], ) for i in range(len(id)) ] all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) connection_all = [] special_k = [] mid_num = 10 for k in range(len(mapIdx)): score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]] candA = all_peaks[limbSeq[k][0] - 1] candB = all_peaks[limbSeq[k][1] - 1] nA = len(candA) nB = len(candB) indexA, indexB = limbSeq[k] if nA != 0 and nB != 0: connection_candidate = [] for i in range(nA): for j in range(nB): vec = np.subtract(candB[j][:2], candA[i][:2]) norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) # failure case when 2 body parts overlaps if norm == 0: continue vec = np.divide(vec, norm) startend = list( zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), np.linspace(candA[i][1], candB[j][1], num=mid_num))) vec_x = np.array( [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \ for I in range(len(startend))]) vec_y = np.array( [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \ for I in range(len(startend))]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply( vec_y, vec[1]) score_with_dist_prior = sum( score_midpts) / len(score_midpts) + min( 0.5 * oriImg.shape[0] / norm - 1, 0) criterion1 = len( np.nonzero(score_midpts > params['thre2']) [0]) > 0.8 * len(score_midpts) criterion2 = score_with_dist_prior > 0 if criterion1 and criterion2: connection_candidate.append([ i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2] ]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) connection = np.zeros((0, 5)) for c in range(len(connection_candidate)): i, j, s = connection_candidate[c][0:3] if i not in connection[:, 3] and j not in connection[:, 4]: connection = np.vstack( [connection, [candA[i][3], candB[j][3], s, i, j]]) if len(connection) >= min(nA, nB): break connection_all.append(connection) else: special_k.append(k) connection_all.append([]) # last number in each row is the total parts number of that person # the second last number in each row is the score of the overall configuration subset = -1 * np.ones((0, 20)) candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(mapIdx)): if k not in special_k: partAs = connection_all[k][:, 0] partBs = connection_all[k][:, 1] indexA, indexB = np.array(limbSeq[k]) - 1 for i in range(len(connection_all[k])): # = 1:size(temp,1) found = 0 subset_idx = [-1, -1] for j in range(len(subset)): # 1:size(subset,1): if subset[j][indexA] == partAs[i] or subset[j][ indexB] == partBs[i]: subset_idx[found] = j found += 1 if found == 1: j = subset_idx[0] if subset[j][indexB] != partBs[i]: subset[j][indexB] = partBs[i] subset[j][-1] += 1 subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] elif found == 2: # if found 2 and disjoint, merge them j1, j2 = subset_idx membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] if len(np.nonzero(membership == 2)[0]) == 0: # merge subset[j1][:-2] += (subset[j2][:-2] + 1) subset[j1][-2:] += subset[j2][-2:] subset[j1][-2] += connection_all[k][i][2] subset = np.delete(subset, j2, 0) else: # as like found == 1 subset[j1][indexB] = partBs[i] subset[j1][-1] += 1 subset[j1][-2] += candidate[ partBs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(20) row[indexA] = partAs[i] row[indexB] = partBs[i] row[-1] = 2 row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \ connection_all[k][i][2] subset = np.vstack([subset, row]) # delete some rows of subset which has few parts occur deleteIdx = [] for i in range(len(subset)): if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: deleteIdx.append(i) subset = np.delete(subset, deleteIdx, axis=0) canvas = cv2.imread(input_image) # B,G,R order print('---------len subset: {}'.format(len(subset))) persons_limbs = [] for n in range(len(subset)): limbs = [] for i in range(17): index = subset[n][np.array(limbSeq[i]) - 1] if -1 in index: limbs.append([]) continue Y = candidate[index.astype(int), 0] X = candidate[index.astype(int), 1] limbs.append([(X[0], Y[0]), (X[1], Y[1])]) # cv2.line(canvas, (int(Y[0]), int(X[0])), (int(Y[1]), int(X[1])), colors[n % 18], 2) persons_limbs.append(limbs) height, width = oriImg.shape[:2] head_pose = HeadPoseEstimator((height, width), mode='nose_eyes_ears') for limbs in persons_limbs: body_pose = geo.BodyPose.from_connected_body_parts(limbs) # geo.draw_body_pose_key_points(body_pose, canvas, colors) # if body_pose.is_hands_up(threshold=30): # print('----------hand up----------') # x1, y1, x2, y2 = body_pose.mbr() # cv2.rectangle(canvas, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2) # cv2.putText(canvas, 'Hand up', (int(x1), int(y1) - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2) # if body_pose.left_elbow is not None: # print('===left_elbow===: {}'.format(body_pose.nose)) # cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.left_elbow), 4, (0, 255, 255), thickness=-1) # # if body_pose.nose is not None: # print('===nose===: {}'.format(body_pose.nose)) # cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.nose), 4, (0, 255, 0), thickness=-1) # if body_pose.left_wrist is not None: # print('===left_wrist===: {}'.format(body_pose.left_wrist)) # cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.left_wrist), 4, (255, 0, 0), thickness=-1) # if body_pose.right_wrist is not None: # print('===right_wrist===: {}'.format(body_pose.right_wrist)) # cv2.circle(canvas, geo.convert_2_int_tuple(body_pose.right_wrist), 4, (0, 0, 255), thickness=-1) body_pose.head_direction(canvas, offset2nose=30) # print(body_pose.nose_2eyes()) # if body_pose.head_key_points() is not None: # rotation_vector, translation_vector = head_pose.solve_pose(body_pose.head_key_points()) # print('r: {}, t: {}'.format(rotation_vector, translation_vector)) # points_2d = head_pose.projection(rotation_vector, translation_vector) # points_2d = np.array(points_2d).astype(np.int).tolist() # print('points: {}'.format(points_2d)) # geo.draw_head_pose(canvas, points_2d) return canvas