def prepare(config, model_file): model = get_testing_model(np_branch1=config.paf_layers, np_branch2=config.heat_layers + 1) print("using model:", model_file) model.load_weights(model_file) return model
def gen_trained_model(): model = get_testing_model() model.compile model.load_weights('weights.0100.h5') model.save('keras_openpose_trained_model.hd5')
default='model/keras/model.h5', help='path to the weights file') args = parser.parse_args() input_image = args.image output = args.output keras_weights_file = args.model tic = time.time() print('start processing...') # load model # authors of original model don't use # vgg normalization (subtracting mean) on input images model = get_testing_model() model.load_weights(keras_weights_file) # load config params, model_params = config_reader() # generate image with body parts canvas = process(input_image, params, model_params) toc = time.time() print('processing time is %.5f' % (toc - tic)) cv2.imwrite(output, canvas) cv2.destroyAllWindows()
def process (input_image, params, model_params): oriImg = cv_imread(input_image) # B,G,R order multiplier = 0.8*model_params['boxsize']/oriImg.shape[0] scale = multiplier ##图片x 缩放到368 imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'],model_params['padValue']) cv2.imwrite("pad.png",imageToTest) input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2)) # required shape (1, width, height, channels) tic = time.time() global flag global model if not flag: model = get_testing_model() model.load_weights('model/keras/model.h5') flag = True toc = time.time() output_blobs = model.predict(input_img) print(toc-tic) #提取输出,调整大小并删除填充 ##heat,ap[i][j][k] 是 (j,i)处为姿势k的置信度 heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) ## paf[i][j][k]表示[j][i]处位于姿势k的概率 paf = np.squeeze(output_blobs[0]) # output 0 is PAFs paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC) paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) all_peaks = []##每个关键点的可能性最大的坐标 peak_counter = 0 for part in range(18): map_ori = heatmap[:, :, part]## 每一个姿势的置信矩阵 map = gaussian_filter(map_ori, sigma=3)## 高斯模糊 map_left = np.zeros(map.shape) map_left[1:, :] = map[:-1, :] map_right = np.zeros(map.shape) map_right[:-1, :] = map[1:, :] map_up = np.zeros(map.shape) map_up[:, 1:] = map[:, :-1] map_down = np.zeros(map.shape) map_down[:, :-1] = map[:, 1:] peaks_binary = np.logical_and.reduce( (map >= map_left, map >= map_right, map >= map_up, map >= map_down, map > params['thre1'])) peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks] id = range(peak_counter, peak_counter + len(peaks)) peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))] all_peaks.append(peaks_with_score_and_id) peak_counter += len(peaks) connection_all = [] special_k = [] mid_num = 10 for k in range(len(mapIdx)): score_mid = paf[:, :, [x - 19 for x in mapIdx[k]]] candA = all_peaks[limbSeq[k][0] - 1]##A点集 candB = all_peaks[limbSeq[k][1] - 1]##B点集 nA = len(candA) nB = len(candB) if (nA != 0 and nB != 0): connection_candidate = [] for i in range(nA): for j in range(nB): vec = np.subtract(candB[j][:2], candA[i][:2]) #两点之间的坐标向量 norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])## 向量的模 if norm == 0: continue vec = np.divide(vec, norm)## 单位向量 ##一个资态向量组,长度为10, startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \ np.linspace(candA[i][1], candB[j][1], num=mid_num))) vec_x = np.array( [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \ for I in range(len(startend))]) vec_y = np.array( [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \ for I in range(len(startend))]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1]) # print(vec[0],vec[1],score_midpts) # score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min( # oriImg.shape[0] / norm - 1, 0) ## A,B之间的平均权值 score_with_dist_prior = sum(score_midpts) / len(score_midpts) ## 80%以上的点的权值大于阈值 criterion1 = len(np.nonzero(score_midpts > params['thre2'])[0]) > 0.8 * len( score_midpts) criterion2 = score_with_dist_prior > 0 if criterion1 and criterion2:## 构造候选图 connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]]) ##按照权值从大到小排序 connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) connection = np.zeros((0, 5)) for c in range(len(connection_candidate)): i, j, s = connection_candidate[c][0:3] if (i not in connection[:, 3] and j not in connection[:, 4]): connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) ##第A类点的第i个和第B类点的第j个之间的权值 if (len(connection) >= min(nA, nB)): break connection_all.append(connection) else: special_k.append(k) connection_all.append([]) subset = -1 * np.ones((0, 20)) candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(mapIdx)): if k not in special_k: partAs = connection_all[k][:, 0] partBs = connection_all[k][:, 1] indexA, indexB = np.array(limbSeq[k]) - 1 for i in range(len(connection_all[k])): found = 0 subset_idx = [-1, -1] for j in range(len(subset)): # 1:size(subset,1): if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]: subset_idx[found] = j found += 1 if found == 1: j = subset_idx[0] if (subset[j][indexB] != partBs[i]): subset[j][indexB] = partBs[i] subset[j][-1] += 1 subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] elif found == 2: # if found 2 and disjoint, merge them j1, j2 = subset_idx membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] if len(np.nonzero(membership == 2)[0]) == 0: # merge subset[j1][:-2] += (subset[j2][:-2] + 1) subset[j1][-2:] += subset[j2][-2:] subset[j1][-2] += connection_all[k][i][2] subset = np.delete(subset, j2, 0) else: # as like found == 1 subset[j1][indexB] = partBs[i] subset[j1][-1] += 1 subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(20) row[indexA] = partAs[i] row[indexB] = partBs[i] row[-1] = 2 row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \ connection_all[k][i][2] subset = np.vstack([subset, row]) deleteIdx = []; for i in range(len(subset)): if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.5: deleteIdx.append(i) subset = np.delete(subset, deleteIdx, axis=0) canvas = cv_imread(input_image) # B,G,R order only_Pose = np.zeros((canvas.shape[0],canvas.shape[1],3), np.uint8) only_Pose.fill(0) maxx=len(all_peaks[0]) for i in range(18): for j in range(len(all_peaks[i])): cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1) cv2.circle(only_Pose,all_peaks[i][j][0:2], 4, colors[i], thickness=-1) #cv2.putText(canvas,"("+str(all_peaks[i][j][0])+","+str(all_peaks[i][j][1])+")",all_peaks[i][j][0:2],cv2.FONT_HERSHEY_SIMPLEX,0.25,(255, 0, 0),1) ##骨架宽度 stickwidth = 2 print(len(subset)) for i in range(17): for n in range(len(subset)): index = subset[n][np.array(limbSeq[i]) - 1] ##第n个人的第i个关键点的index if -1 in index: continue cur_canvas = canvas.copy() cur_pose = only_Pose.copy() Y = candidate[index.astype(int), 0] X = candidate[index.astype(int), 1] # print(n,i,Y[0],X[0],Y[1],X[1]) ##第n个人第i个肢体的向量 mX = np.mean(X) mY = np.mean(Y) length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) cv2.fillConvexPoly(cur_pose, polygon, colors[i]) canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) only_Pose = cv2.addWeighted(only_Pose, 0.4, cur_pose,0.6,0) ##给每个人分配id for n in range(len(subset)): index = subset[n][np.array(limbSeq[1]) - 1] if -1 in index: index = subset[n][np.array(limbSeq[2]) - 1] Y = candidate[index.astype(int), 0] X = candidate[index.astype(int), 1] #print(Y[0],X[0]) cv2.putText(canvas,str(n+1),(int(Y[0]),int(X[0]-5)),cv2.FONT_HERSHEY_SIMPLEX,1.5,(68,255, 51),5) person_count = len(subset) cv2.imwrite("only_Pose.png", only_Pose) cv2.imwrite("result.png", canvas) print(canvas.shape) return canvas,person_count
parser.add_argument('--output', type=str, default='result.png', help='output image') parser.add_argument('--model', type=str, default='model/keras/model.h5', help='path to the weights file') args = parser.parse_args() input_image = args.image output = args.output keras_weights_file = args.model tic = time.time() print('start processing...') # load model # authors of original model don't use # vgg normalization (subtracting mean) on input images model = get_testing_model() model.load_weights(keras_weights_file) # load config params, model_params = config_reader() # generate image with body parts canvas = process(input_image, params, model_params) toc = time.time() print ('processing time is %.5f' % (toc - tic)) cv2.imwrite(output, canvas) cv2.destroyAllWindows()
funcdict = { 'image_file': detect_image_file, 'video_file': detect_video_file, 'video_folder': detect_video_folder } #........................................................................... #loading data from configuration file confs = json.load(open('pose_configuration.json')) tic = time.time() print('start processing...') # load model model = get_testing_model(vgg_norm=False) model.load_weights(keras_weights_file) # load config params, model_params = config_reader() # calling main function funcdict[confs['source_type']](info=confs[confs['source_type']], params=params, model_params=model_params) toc = time.time() print('processing time is %.5f' % (toc - tic)) import argparse import cv2
def get_prediction(keras_weights_file, model_name, save_images, image_path): image_save_path = os.path.join('../data/pose_model/images', model_name) if os.path.isdir(image_save_path) != 1: os.mkdir(image_save_path) image_list = glob.glob(os.path.join(image_path, '*')) print('loading model') model = get_testing_model() print('loading weights') model.load_weights(keras_weights_file) print('config') params, model_params = config_reader() df = pd.DataFrame() print('processing images') for input_image in image_list: image_id = os.path.basename(input_image)[:-4] video_number = image_id[1:7] frame_number = image_id[7:] try: output_dict = process(input_image, params, model_params, model) frame = output_dict['canvas'] if save_images == 1: cv2.imwrite(os.path.join(image_save_path, image_id + '.jpg'), frame) del output_dict['canvas'] output_dict.update({ 'file_path': input_image, 'video': video_number, 'frame': frame_number, 'id': int(image_id) }) output_df = pd.DataFrame(pd.Series(output_dict)).transpose() # save frame as image df = df.append(output_df) except: print('error during pose estimation') # df with idx and person idx kp_pred_df = df.reset_index().groupby('id').apply(get_skel).reset_index() # take person_idx with the most keypoints counts = kp_pred_df.groupby(['id', 'person_idx'])['c'].count().reset_index() max_rows = counts.groupby('id')['c'].idxmax().tolist() max_rows_df = counts.loc[max_rows, ['id', 'person_idx']] max_rows_df['dum'] = 1 kp_pred_df = pd.merge(kp_pred_df, max_rows_df, on=['id', 'person_idx'], how='inner') kp_pred_df = kp_pred_df[['id', 'x_pred', 'y_pred', 'part_idx']] # 'c', # add part labels COCO_label_df = pd.Series(['nose', 'neck', 'right_shoulder', 'right_elbow','right_wrist',\ 'left_shoulder','left_elbow','left_wrist','right_hip','right_knee','right_ankle',\ 'left_hip','left_knee','left_ankle','right_eye','left_eye','right_ear','left_ear']).reset_index() COCO_label_df.columns = ['part_idx', 'part_label'] kp_pred_df = pd.merge(kp_pred_df, COCO_label_df, on='part_idx', how='left') # x,y: replace zeros with nans kp_pred_df.loc[kp_pred_df.x_pred == 0, 'x_pred'] = np.nan kp_pred_df.loc[kp_pred_df.y_pred == 0, 'y_pred'] = np.nan columns = pd.MultiIndex.from_product([[model_name], ['x', 'y']], names=['var_type', 'dim']) index = pd.MultiIndex.from_arrays( [np.array(kp_pred_df.id), np.array(kp_pred_df.part_label)], names=['id', 'part_label']) kp_pred_df = pd.DataFrame(kp_pred_df[['x_pred', 'y_pred']].as_matrix(), index=index, columns=columns) return kp_pred_df
lstImages = glob.glob('{}/*[0-9].jpg'.format(args.imdir)) numImages = len(lstImages) for ii, input_image in enumerate(lstImages): timg = cv2.imread(input_image) if timg is None: print('\t!!! WARNING !!! Image is invalid, skip ... [{}]'.format( input_image)) continue tic = time.time() print('start processing...') # load model model = get_testing_model(pinpShape=(368, 368, 3)) model.summary() model.load_weights(keras_weights_file) model.summary() params, model_params = config_reader() # generate image with body parts canvas = process(input_image, params, model_params) toc = time.time() print('processing time is %.3f (s)' % (toc - tic)) plt.subplot(1, 2, 1) plt.imshow(plt.imread(input_image)) plt.subplot(1, 2, 2) plt.imshow(canvas)
def __init__(self, weights_file): self.model = get_testing_model() self.model.load_weights(weights_file) self.im_height = 0 self.im_width = 0
parser.add_argument('--output', type=str, default='result.png', help='output image') parser.add_argument('--model', type=str, default='model/keras/model.h5', help='path to the weights file') args = parser.parse_args() input_image = args.image output = args.output keras_weights_file = args.model tic = time.time() print('start processing...') # load model # authors of original model don't use # vgg normalization (subtracting mean) on input images model = get_testing_model(1,1) model.load_weights(keras_weights_file) # load config params, model_params = config_reader() # generate image with body parts canvas = process(input_image, params, model_params) toc = time.time() print ('processing time is %.5f' % (toc - tic)) cv2.imwrite(output, canvas) cv2.destroyAllWindows()
def compute_keypoints(model_weights_file, cocoGt, coco_api_dir, coco_data_type, eval_method, epoch_num): # load model model = get_testing_model() model.load_weights(model_weights_file) # load model config params, model_params = config_reader() # load epoch num trained_epoch = epoch_num # load validation image ids imgIds = sorted(cocoGt.getImgIds()) # eval model mode_name = '' if eval_method == 1: mode_name = 'open-pose-multi-scale' elif eval_method == 0: mode_name = 'open-pose-single-scale' # prepare json output json_file = open(args.outputjson, 'w') if not os.path.exists('./results'): os.mkdir('./results') output_folder = './results/val2014-ours-epoch%d-%s' % (trained_epoch, mode_name) if not os.path.exists(output_folder): os.mkdir(output_folder) prediction_folder = '%s/predictions' % (output_folder) if not os.path.exists(prediction_folder): os.mkdir(prediction_folder) # prepare json output json_fpath = '%s/%s' % (output_folder, args.outputjson) json_file = open(json_fpath, 'w') candidate_set = [] subset_set = [] image_id_set = [] counter = 0 # run keypoints detection per image for item in imgIds: # load image fname fname = cocoGt.imgs[item]['file_name'] input_fname = '../dataset/%s/%s' % (coco_data_type, fname) print(input_fname) print('Image file exist? %s' % os.path.isfile(input_fname)) # run keypoint detection if eval_method == 1: visual_result, candidate, subset = process_multi_scale( input_fname, model, params, model_params) elif eval_method == 0: visual_result, candidate, subset = process_single_scale( input_fname, model, params, model_params) # draw results output_fname = '%s/result_%s' % (prediction_folder, fname) cv2.imwrite(output_fname, visual_result) candidate_set.append(candidate) subset_set.append(subset) image_id_set.append(item) counter = counter + 1 # dump results to json file write_json(candidate_set, subset_set, image_id_set, json_file) return json_fpath
def __init__(self, model_path = 'model/keras/model.h5'): self.model = get_testing_model() self.model.load_weights(model_path) self.params, self.model_params = config_reader() self.image = None