def prepare(config, model_file):
model = get_testing_model(np_branch1=config.paf_layers,
np_branch2=config.heat_layers + 1)
print("using model:", model_file)
model.load_weights(model_file)
return model
def gen_trained_model():
model = get_testing_model()
model.compile
model.load_weights('weights.0100.h5')
model.save('keras_openpose_trained_model.hd5')
default='model/keras/model.h5',
help='path to the weights file')
args = parser.parse_args()
input_image = args.image
output = args.output
keras_weights_file = args.model
tic = time.time()
print('start processing...')
# load model
# authors of original model don't use
# vgg normalization (subtracting mean) on input images
model = get_testing_model()
model.load_weights(keras_weights_file)
# load config
params, model_params = config_reader()
# generate image with body parts
canvas = process(input_image, params, model_params)
toc = time.time()
print('processing time is %.5f' % (toc - tic))
cv2.imwrite(output, canvas)
cv2.destroyAllWindows()
def process (input_image, params, model_params):
oriImg = cv_imread(input_image) # B,G,R order
multiplier = 0.8*model_params['boxsize']/oriImg.shape[0]
scale = multiplier
##图片x 缩放到368
imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'],model_params['padValue'])
cv2.imwrite("pad.png",imageToTest)
input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2)) # required shape (1, width, height, channels)
tic = time.time()
global flag
global model
if not flag:
model = get_testing_model()
model.load_weights('model/keras/model.h5')
flag = True
toc = time.time()
output_blobs = model.predict(input_img)
print(toc-tic)
#提取输出，调整大小并删除填充
##heat,ap[i][j][k] 是 (j,i)处为姿势k的置信度
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3],
:]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
## paf[i][j][k]表示[j][i]处位于姿势k的概率
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'],
interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
all_peaks = []##每个关键点的可能性最大的坐标
peak_counter = 0
for part in range(18):
map_ori = heatmap[:, :, part]## 每一个姿势的置信矩阵
map = gaussian_filter(map_ori, sigma=3)## 高斯模糊
map_left = np.zeros(map.shape)
map_left[1:, :] = map[:-1, :]
map_right = np.zeros(map.shape)
map_right[:-1, :] = map[1:, :]
map_up = np.zeros(map.shape)
map_up[:, 1:] = map[:, :-1]
map_down = np.zeros(map.shape)
map_down[:, :-1] = map[:, 1:]
peaks_binary = np.logical_and.reduce(
(map >= map_left, map >= map_right, map >= map_up, map >= map_down, map > params['thre1']))
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf[:, :, [x - 19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0] - 1]##A点集
candB = all_peaks[limbSeq[k][1] - 1]##B点集
nA = len(candA)
nB = len(candB)
if (nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2]) #两点之间的坐标向量
norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])## 向量的模
if norm == 0:
continue
vec = np.divide(vec, norm)## 单位向量
##一个资态向量组，长度为10，
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
vec_x = np.array(
[score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array(
[score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
# print(vec[0],vec[1],score_midpts)
# score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min(
# oriImg.shape[0] / norm - 1, 0)
## A,B之间的平均权值
score_with_dist_prior = sum(score_midpts) / len(score_midpts)
## 80%以上的点的权值大于阈值
criterion1 = len(np.nonzero(score_midpts > params['thre2'])[0]) > 0.8 * len(
score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:## 构造候选图
connection_candidate.append([i, j, score_with_dist_prior,
score_with_dist_prior + candA[i][2] + candB[j][2]])
##按照权值从大到小排序
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0, 5))
for c in range(len(connection_candidate)):
i, j, s = connection_candidate[c][0:3]
if (i not in connection[:, 3] and j not in connection[:, 4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) ##第A类点的第i个和第B类点的第j个之间的权值
if (len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:, 0]
partBs = connection_all[k][:, 1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])):
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): # 1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if (subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: # merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \
connection_all[k][i][2]
subset = np.vstack([subset, row])
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.5:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
canvas = cv_imread(input_image) # B,G,R order
only_Pose = np.zeros((canvas.shape[0],canvas.shape[1],3), np.uint8)
only_Pose.fill(0)
maxx=len(all_peaks[0])
for i in range(18):
for j in range(len(all_peaks[i])):
cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
cv2.circle(only_Pose,all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
#cv2.putText(canvas,"("+str(all_peaks[i][j][0])+","+str(all_peaks[i][j][1])+")",all_peaks[i][j][0:2],cv2.FONT_HERSHEY_SIMPLEX,0.25,(255, 0, 0),1)
##骨架宽度
stickwidth = 2
print(len(subset))
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i]) - 1] ##第n个人的第i个关键点的index
if -1 in index:
continue
cur_canvas = canvas.copy()
cur_pose = only_Pose.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
# print(n,i,Y[0],X[0],Y[1],X[1]) ##第n个人第i个肢体的向量
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
cv2.fillConvexPoly(cur_pose, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
only_Pose = cv2.addWeighted(only_Pose, 0.4, cur_pose,0.6,0)
##给每个人分配id
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[1]) - 1]
if -1 in index:
index = subset[n][np.array(limbSeq[2]) - 1]
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
#print(Y[0],X[0])
cv2.putText(canvas,str(n+1),(int(Y[0]),int(X[0]-5)),cv2.FONT_HERSHEY_SIMPLEX,1.5,(68,255, 51),5)
person_count = len(subset)
cv2.imwrite("only_Pose.png", only_Pose)
cv2.imwrite("result.png", canvas)
print(canvas.shape)
return canvas,person_count
parser.add_argument('--output', type=str, default='result.png', help='output image')
parser.add_argument('--model', type=str, default='model/keras/model.h5', help='path to the weights file')
args = parser.parse_args()
input_image = args.image
output = args.output
keras_weights_file = args.model
tic = time.time()
print('start processing...')
# load model
# authors of original model don't use
# vgg normalization (subtracting mean) on input images
model = get_testing_model()
model.load_weights(keras_weights_file)
# load config
params, model_params = config_reader()
# generate image with body parts
canvas = process(input_image, params, model_params)
toc = time.time()
print ('processing time is %.5f' % (toc - tic))
cv2.imwrite(output, canvas)
cv2.destroyAllWindows()
funcdict = {
'image_file': detect_image_file,
'video_file': detect_video_file,
'video_folder': detect_video_folder
}
#...........................................................................
#loading data from configuration file
confs = json.load(open('pose_configuration.json'))
tic = time.time()
print('start processing...')
# load model
model = get_testing_model(vgg_norm=False)
model.load_weights(keras_weights_file)
# load config
params, model_params = config_reader()
# calling main function
funcdict[confs['source_type']](info=confs[confs['source_type']],
params=params,
model_params=model_params)
toc = time.time()
print('processing time is %.5f' % (toc - tic))
import argparse
import cv2
def get_prediction(keras_weights_file, model_name, save_images, image_path):
image_save_path = os.path.join('../data/pose_model/images', model_name)
if os.path.isdir(image_save_path) != 1:
os.mkdir(image_save_path)
image_list = glob.glob(os.path.join(image_path, '*'))
print('loading model')
model = get_testing_model()
print('loading weights')
model.load_weights(keras_weights_file)
print('config')
params, model_params = config_reader()
df = pd.DataFrame()
print('processing images')
for input_image in image_list:
image_id = os.path.basename(input_image)[:-4]
video_number = image_id[1:7]
frame_number = image_id[7:]
try:
output_dict = process(input_image, params, model_params, model)
frame = output_dict['canvas']
if save_images == 1:
cv2.imwrite(os.path.join(image_save_path, image_id + '.jpg'),
frame)
del output_dict['canvas']
output_dict.update({
'file_path': input_image,
'video': video_number,
'frame': frame_number,
'id': int(image_id)
})
output_df = pd.DataFrame(pd.Series(output_dict)).transpose()
# save frame as image
df = df.append(output_df)
except:
print('error during pose estimation')
# df with idx and person idx
kp_pred_df = df.reset_index().groupby('id').apply(get_skel).reset_index()
# take person_idx with the most keypoints
counts = kp_pred_df.groupby(['id',
'person_idx'])['c'].count().reset_index()
max_rows = counts.groupby('id')['c'].idxmax().tolist()
max_rows_df = counts.loc[max_rows, ['id', 'person_idx']]
max_rows_df['dum'] = 1
kp_pred_df = pd.merge(kp_pred_df,
max_rows_df,
on=['id', 'person_idx'],
how='inner')
kp_pred_df = kp_pred_df[['id', 'x_pred', 'y_pred', 'part_idx']] # 'c',
# add part labels
COCO_label_df = pd.Series(['nose', 'neck', 'right_shoulder', 'right_elbow','right_wrist',\
'left_shoulder','left_elbow','left_wrist','right_hip','right_knee','right_ankle',\
'left_hip','left_knee','left_ankle','right_eye','left_eye','right_ear','left_ear']).reset_index()
COCO_label_df.columns = ['part_idx', 'part_label']
kp_pred_df = pd.merge(kp_pred_df, COCO_label_df, on='part_idx', how='left')
# x,y: replace zeros with nans
kp_pred_df.loc[kp_pred_df.x_pred == 0, 'x_pred'] = np.nan
kp_pred_df.loc[kp_pred_df.y_pred == 0, 'y_pred'] = np.nan
columns = pd.MultiIndex.from_product([[model_name], ['x', 'y']],
names=['var_type', 'dim'])
index = pd.MultiIndex.from_arrays(
[np.array(kp_pred_df.id),
np.array(kp_pred_df.part_label)],
names=['id', 'part_label'])
kp_pred_df = pd.DataFrame(kp_pred_df[['x_pred', 'y_pred']].as_matrix(),
index=index,
columns=columns)
return kp_pred_df
lstImages = glob.glob('{}/*[0-9].jpg'.format(args.imdir))
numImages = len(lstImages)
for ii, input_image in enumerate(lstImages):
timg = cv2.imread(input_image)
if timg is None:
print('\t!!! WARNING !!! Image is invalid, skip ... [{}]'.format(
input_image))
continue
tic = time.time()
print('start processing...')
# load model
model = get_testing_model(pinpShape=(368, 368, 3))
model.summary()
model.load_weights(keras_weights_file)
model.summary()
params, model_params = config_reader()
# generate image with body parts
canvas = process(input_image, params, model_params)
toc = time.time()
print('processing time is %.3f (s)' % (toc - tic))
plt.subplot(1, 2, 1)
plt.imshow(plt.imread(input_image))
plt.subplot(1, 2, 2)
plt.imshow(canvas)
def __init__(self, weights_file):
self.model = get_testing_model()
self.model.load_weights(weights_file)
self.im_height = 0
self.im_width = 0
parser.add_argument('--output', type=str, default='result.png', help='output image')
parser.add_argument('--model', type=str, default='model/keras/model.h5', help='path to the weights file')
args = parser.parse_args()
input_image = args.image
output = args.output
keras_weights_file = args.model
tic = time.time()
print('start processing...')
# load model
# authors of original model don't use
# vgg normalization (subtracting mean) on input images
model = get_testing_model(1,1)
model.load_weights(keras_weights_file)
# load config
params, model_params = config_reader()
# generate image with body parts
canvas = process(input_image, params, model_params)
toc = time.time()
print ('processing time is %.5f' % (toc - tic))
cv2.imwrite(output, canvas)
cv2.destroyAllWindows()
def compute_keypoints(model_weights_file, cocoGt, coco_api_dir, coco_data_type,
eval_method, epoch_num):
# load model
model = get_testing_model()
model.load_weights(model_weights_file)
# load model config
params, model_params = config_reader()
# load epoch num
trained_epoch = epoch_num
# load validation image ids
imgIds = sorted(cocoGt.getImgIds())
# eval model
mode_name = ''
if eval_method == 1:
mode_name = 'open-pose-multi-scale'
elif eval_method == 0:
mode_name = 'open-pose-single-scale'
# prepare json output
json_file = open(args.outputjson, 'w')
if not os.path.exists('./results'):
os.mkdir('./results')
output_folder = './results/val2014-ours-epoch%d-%s' % (trained_epoch,
mode_name)
if not os.path.exists(output_folder):
os.mkdir(output_folder)
prediction_folder = '%s/predictions' % (output_folder)
if not os.path.exists(prediction_folder):
os.mkdir(prediction_folder)
# prepare json output
json_fpath = '%s/%s' % (output_folder, args.outputjson)
json_file = open(json_fpath, 'w')
candidate_set = []
subset_set = []
image_id_set = []
counter = 0
# run keypoints detection per image
for item in imgIds:
# load image fname
fname = cocoGt.imgs[item]['file_name']
input_fname = '../dataset/%s/%s' % (coco_data_type, fname)
print(input_fname)
print('Image file exist? %s' % os.path.isfile(input_fname))
# run keypoint detection
if eval_method == 1:
visual_result, candidate, subset = process_multi_scale(
input_fname, model, params, model_params)
elif eval_method == 0:
visual_result, candidate, subset = process_single_scale(
input_fname, model, params, model_params)
# draw results
output_fname = '%s/result_%s' % (prediction_folder, fname)
cv2.imwrite(output_fname, visual_result)
candidate_set.append(candidate)
subset_set.append(subset)
image_id_set.append(item)
counter = counter + 1
# dump results to json file
write_json(candidate_set, subset_set, image_id_set, json_file)
return json_fpath
def __init__(self, model_path = 'model/keras/model.h5'):
self.model = get_testing_model()
self.model.load_weights(model_path)
self.params, self.model_params = config_reader()
self.image = None
