def opImages_2_json(imagePaths, params):
"""Applique openpose selon les paramètres params sur les images de la liste
imagePaths. Le résultat est en format JSON au path params["write_json"] """
if params["write_json"] == "":
raise ValueError(
"Le chemin d'écriture ne peut pas être vide. Définissez vos paramètres avec un chemin d'écriture pour write_json"
)
opWrapper = init_openpose(params)
start = time.time()
for imageId in tqdm(range(len(imagePaths))):
imagePath = imagePaths[imageId]
datum = op.Datum()
image = cv2.imread(imagePath)
datum.cvInputData = image
if params["hand_detector"] == 2:
imageSize = max(image.shape[0:2])
handRectangles = [[
op.Rectangle(0., 0., imageSize, imageSize),
op.Rectangle(0., 0., imageSize, imageSize),
]]
datum.handRectangles = handRectangles
opWrapper.emplaceAndPop([datum])
end = time.time()
print("OpenPose demo successfully finished. Total time: " +
str(end - start) + " seconds")
def detect_key_points(video_to_process, op_wrapper):
"""
Creates rectangle and process it
:param op_wrapper: open pose wrapper object
:param video_to_process: open_cv object to process
:return: np.array with data
"""
width = video_to_process.get(3) # float width of video
height = video_to_process.get(4) # float high of video
# by default rectangle for right hand by center
right_hand_rectangle = [
# center
[
op.Rectangle(0., 0., 0., 0.),
op.Rectangle(0.2 * width, 0.1 * height, height * 0.85,
height * 0.85)
]
]
datum = op.Datum()
data = None
timestamps = []
while video_to_process.isOpened():
# Capture frame-by-frame
ret, frame = video_to_process.read()
if ret:
datum.cvInputData = frame
datum.handRectangles = right_hand_rectangle
op_wrapper.emplaceAndPop([datum])
cv2.imshow("OpenPose 1.4.0. Right hand keypoints",
datum.cvOutputData)
timestamps.append(video_to_process.get(cv2.CAP_PROP_POS_MSEC))
if data is not None:
data = np.append(data,
datum.handKeypoints[1].reshape(1, 63),
axis=0)
else:
data = datum.handKeypoints[1].reshape(1, 63)
# Press Q on keyboard to exit
if cv2.waitKey(25) & 0xFF == ord('q'):
break
# Break the loop
else:
break
# add timestamps to beginning
data_with_time = np.append(np.array(timestamps).reshape(
len(timestamps), 1),
data,
axis=1)
return data_with_time
def opFrame_run(frame, params, opWrapper):
"""Applique openpose sur une frame et retourne l'object datum contenant :
- l'image avec le squelette
- les keypoints en numpy
"""
datum = op.Datum()
datum.cvInputData = frame
if params["hand_detector"] == 2:
imageSize = float(max(frame.shape[0:2]))
handRectangles = [[
op.Rectangle(0., 0., imageSize, imageSize),
op.Rectangle(0., 0., imageSize, imageSize),
]]
datum.handRectangles = handRectangles
opWrapper.emplaceAndPop([datum])
return datum
def box2oprectangle(box):
left, right, top, bottom = box
width = np.abs(right - left)
height = np.abs(top - bottom)
max_length = int(max(width,height))
center = (int(left + (width /2 )), int(bottom - (height/2)))
# Openpose hand detector needs the bounding box to be quite big , so we make it bigger
# Top point for rectangle
new_top = (int(center[0] - max_length / 1.3), int(center[1] - max_length /1.3))
max_length = int(max_length * 1.6)
hand_rectangle = op.Rectangle(new_top[0],new_top[1],max_length,max_length)
return hand_rectangle
def predict_keypoints(color_image,rect):
imageToProcess = color_image
if rect is None:
rect = [100.,50.,400.,400.]
handRectangles = [
# Left/Right hands person 0
[
op.Rectangle(0., 0., 0., 0.),
op.Rectangle(rect[0], rect[1], rect[2], rect[3]),
]
]
# Create new datum
datum.cvInputData = imageToProcess
datum.handRectangles = handRectangles
# Process and display image
opWrapper.emplaceAndPop([datum])
# Debug code
# print("Left hand keypoints: \n" + str(datum.handKeypoints[0]))
# print("Right hand keypoints: \n" + str(datum.handKeypoints[1]))
# cv2.imshow("OpenPose 1.5.1 - Tutorial Python API", datum.cvOutputData)
return(datum.handKeypoints[1],datum.cvOutputData)
def opImage_display(imagePath, params, display=True, frame=None):
"""Applique openpose sur une image et retourne l'object datum contenant :
- l'image avec le squelette
- les keypoints en numpy
"""
opWrapper = init_openpose(params)
datum = op.Datum()
if frame:
image = frame
else:
image = cv2.imread(imagePath)
datum.cvInputData = image
if params["hand_detector"] == 2:
imageSize = float(max(image.shape[0:2]))
handRectangles = [[
op.Rectangle(0., 0., imageSize, imageSize),
op.Rectangle(0., 0., imageSize, imageSize),
]]
datum.handRectangles = handRectangles
opWrapper.emplaceAndPop([datum])
if display:
plt.imshow(datum.cvOutputData)
plt.show()
return datum
def detect_keypoints(image, hand_boxes, threshold=0.5):
# We are considering every seen hand is a left hand
hands_rectangles = [[box2oprectangle(box),op.Rectangle(0., 0., 0., 0.)] for box in hand_boxes]
# hands_rectangles.append([hand_rectangle,op.Rectangle(0., 0., 0., 0.)])
# Create new datum
datum = op.Datum()
datum.cvInputData = image
datum.handRectangles = hands_rectangles
# Process and display image
opWrapper.emplaceAndPop([datum])
if datum.handKeypoints[0].shape == ():
# if there were no detections
hand_keypoints = [[],[]]
else:
hand_keypoints = datum.handKeypoints
return hand_keypoints, datum.cvOutputData
def get_hand_rectangle_from_keypoints(wrist, elbow, shoulder) -> op.Rectangle:
"""
Adapted from original CPP code:
https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp
"""
ratio_wrist_elbow = 0.33 # Taken from openpose
hand_rectangle = op.Rectangle()
hand_rectangle.x = wrist[0] + ratio_wrist_elbow * (wrist[0] - elbow[0])
hand_rectangle.y = wrist[1] + ratio_wrist_elbow * (wrist[1] - elbow[1])
distance_wrist_elbow = get_distance(wrist, elbow)
distance_elbow_shoulder = get_distance(elbow, shoulder)
hand_rectangle.width = 1.5 * np.max(
[distance_wrist_elbow, 0.9 * distance_elbow_shoulder])
hand_rectangle.height = hand_rectangle.width
hand_rectangle.x -= hand_rectangle.width / 2.
hand_rectangle.y -= hand_rectangle.width / 2.
return hand_rectangle
params["model_folder"] = '../data/models/'
# Do not use the face detector.
# Define constant face rectangles.
params["face_detector"] = 2
params["face"] = True
params["body"] = 0
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
# Face area in the video
face_rects = [
op.Rectangle(400, 30, 512, 512),
]
def op_process_file(filename):
img = cv2.imread(filename)
datum = op.Datum()
datum.cvInputData = img
datum.faceRectangles = face_rects
opWrapper.emplaceAndPop([datum])
keypoints = datum.faceKeypoints[0]
return keypoints
# Add others in path?
for i in range(0, len(args[1])):
curr_item = args[1][i]
if i != len(args[1]) - 1: next_item = args[1][i + 1]
else: next_item = "1"
if "--" in curr_item and "--" in next_item:
key = curr_item.replace('-', '')
if key not in params: params[key] = "1"
elif "--" in curr_item and "--" not in next_item:
key = curr_item.replace('-', '')
if key not in params: params[key] = next_item
# Construct it from system arguments
# Fixed the handRectangles to only 1 person and 1 big rectangle, don't have to keep changing rectangle
handRectangles = [[
op.Rectangle(100.0, 150.0, 328.0, 390.0),
op.Rectangle(0., 0., 0., 0.)
]]
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
# Process Image
datum = op.Datum()
datum.handRectangles = handRectangles
cam = cv2.VideoCapture(0) # modify here for camera number
while (cv2.waitKey(1) != 27):
# Get camera frame
ret, frame = cam.read()
datum.cvInputData = frame
)
raise e
params = dict()
params["model_folder"] = r'C:\Work\Projects\fyp\openpose\openpose\models'
params["hand"] = True
params["hand_detector"] = 2
params["body"] = 0
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
handRectangles = [
# Left/Right hands person 0
[op.Rectangle(0., 0., 0., 0.),
op.Rectangle(0., 0., res, res)]
]
datum = op.Datum()
datum.handRectangles = handRectangles
cap = cv2.VideoCapture(1)
while True:
# Capture frame-by-frame
ret, frame = cap.read()
frame = cv2.resize(frame, (int(res), int(res)),
interpolation=cv2.INTER_AREA)
# todo remove
top = facepos.item()['top']
bottom = facepos.item()['bottom']
lrdiff = right - left
btdiff = bottom - top
len = 0
x = left
y = top
if lrdiff < btdiff:
len = lrdiff
y = y + (btdiff - lrdiff) / 2
else:
len = btdiff
x = x + (lrdiff - btdiff) / 2
faceRectangles = [
op.Rectangle(x, y, len, len),
]
# Create new datum
datum = op.Datum()
datum.cvInputData = imageToProcess
datum.faceRectangles = faceRectangles
# Process and display image
opWrapper.emplaceAndPop(op.VectorDatum([datum]))
# print("Face keypoints: \n" + str(datum.faceKeypoints))
# save openpose data
# rootForFacepos = root.replace('iphone_pictures', 'openpose')
# if not isdir(rootForFacepos):
# os.makedirs(rootForFacepos)
datum = op.Datum()
# 시간측정
#손,코 데이터 불러옴
with open('./step1/hndata.pickle.', 'rb') as fr:
hnData = pickle.load(fr)
imagePaths = list(paths.list_images('./step2'))
print(imagePaths)
start = time.time()
for idx, imagePath in enumerate(imagePaths):
imageToProcess = imread(imagePath)
filename = imagePath.split(os.path.sep)[-1]
faceRectangles = [
op.Rectangle(hnData[filename][4] - 100.0, hnData[filename][5] - 100.0,
200.0, 200.0),
]
datum.cvInputData = imageToProcess
datum.faceRectangles = faceRectangles
opWrapper.emplaceAndPop([datum])
# 이미지 출력
# cv2.imshow("OpenPose 1.6.0 - Tutorial Python API", datum.cvOutputData)
# cv2.waitKey(0)
# 이미지 저장
# for imagePath in imagePaths:
label = imagePath.split(os.path.sep)[-2]
check_folder('./step3', label)
print('step3', idx, imagePath)
print('time : ', time.time() - start)
def main():
try:
# Import Openpose (Windows/Ubuntu/OSX)
dir_path = os.path.dirname(os.path.realpath(__file__))
try:
# Windows Import
if platform == "win32":
# Change these variables to point to the correct folder (Release/x64 etc.)
sys.path.append(dir_path + '/../../python/openpose/Release')
os.environ['PATH'] = os.environ[
'PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;'
import pyopenpose as op
# from openpose import *
else:
# Change these variables to point to the correct folder (Release/x64 etc.)
sys.path.append('../../python')
# If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it.
# sys.path.append('/usr/local/python')
from openpose import pyopenpose as op
# from openpose import *
except ImportError as e:
print(
'Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?'
)
raise e
# Custom Params (refer to include/openpose/flags.hpp for more parameters)
params = dict()
params["model_folder"] = "../../../models/"
params["hand"] = True
params["hand_detector"] = 2
params["body"] = 1
params["net_resolution"] = '320x192' #20*11
# params["face"] = True
# params["disable_blending"] = True
# params["fps_max"] = 5
handRectangles = [[
op.Rectangle(128, 0, 1024, 1024),
op.Rectangle(0., 0., 0., 0.)
]]
# device = 'cuda' if torch.cuda.is_available() else 'cpu'
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
# Process Image
datum = op.Datum()
datum.handRectangles = handRectangles
cam = cv2.VideoCapture(0) # modify here for camera number
cam.set(3, 1280)
cam.set(4, 1024)
pair_poseKeypoints = [[], []]
input_hands = []
prev_state = None
msg_state = ('not_sent', time.perf_counter())
while (cv2.waitKey(1) != 27):
if msg_state[0] == 'sent':
if time.perf_counter() - msg_state[1] > 2.5:
msg_state = ('not_sent', time.perf_counter())
ret, frame = cam.read()
datum.cvInputData = frame
opWrapper.emplaceAndPop([datum])
frame = datum.cvOutputData
'''If Person not in Camera'''
if datum.poseKeypoints.shape == ():
# conn.request("POST", "/v2/chat/users/[email protected]/messages", payload, headers)
#
# res = conn.getresponse()
# data = res.read()
#
# print(data.decode("utf-8"))
if msg_state[0] == 'not_sent':
# print('WHY NOT WORKING')
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_absent, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 3
fontColor = (255, 255, 0)
lineType = 2
fontThickness = 2
msg_on_screen = 'ABSENT!'
textsize = cv2.getTextSize(msg_on_screen, font, fontScale,
fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
cv2.rectangle(frame, (0, 0), (1280, 1024), (0, 0, 255), 20)
cv2.putText(frame, 'Absent!', bottomLeftCornerOfText, font,
fontScale, fontColor, lineType)
cv2.imshow("Openpose 1.4.0 Webcam", frame)
continue
'''Evaluate Movement & Confidence'''
del pair_poseKeypoints[0]
pair_poseKeypoints.append(datum.poseKeypoints[0])
body_confidence_avg = avg_pose_confidence(datum.poseKeypoints[0])
# print(body_confidence_avg)
moved = metric(pair_poseKeypoints)
'''Evaluate Hand Gesture'''
if len(input_hands) == 12:
del input_hands[0]
input_hands.append(datum.handKeypoints[0][0])
# print(len(input_hands))
prob, gesture = None, None
hand_confidence_avg = avg_list_confidence(input_hands)
# if len(input_hands) == 12 and avg >= 0.1:
if len(input_hands) == 12:
# print('Confidence : ', hand_confidence_avg)
prob, gesture = get_hand_gesture('normalizev2.pt', input_hands,
'cuda')
# print(prob, gesture)
'''Output Recognition Results'''
print_msg = False
font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCornerOfText = None
fontColor = None
fontScale = 3
fontThickness = 2
msg_on_screen = None
if valid_hand(hand_confidence_avg, gesture) and gesture == 1:
print('THUMBS DOWN PROB : ', prob)
if prob > 11:
'''Counter'''
if prev_state is None:
prev_state = ('thumbs_down', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 5.5:
prev_state = ('thumbs_down', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'thumbs_down':
prev_state = ('thumbs_down', time.perf_counter())
# print(prev_state)
else:
# print(time.perf_counter() - prev_state[1])
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_thumbsdown, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
if time.perf_counter() - prev_state[1] > 0.5:
print_msg = True
# bottomLeftCornerOfText = (450, 500)
fontColor = (255, 0, 0)
fontScale = 3
msg_on_screen = 'THUMBS DOWN'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale,
fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) //
2,
(1024 + textsize[1]) //
2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
elif valid_hand(hand_confidence_avg, gesture) and gesture == 2:
print('THUMBS DOWN PROB : ', prob)
'''Counter'''
if prob > 11:
if prev_state is None:
prev_state = ('thumbs up', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 5.5:
prev_state = ('thumbs_up', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'thumbs_up':
prev_state = ('thumbs_up', time.perf_counter())
# print(prev_state)
else:
# print(time.perf_counter() - prev_state[1])
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_thumbsup, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
if time.perf_counter() - prev_state[1] > 0.5:
print_msg = True
# bottomLeftCornerOfText = (450, 500)
fontColor = (0, 255, 0)
fontScale = 3
msg_on_screen = 'THUMBS UP'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale,
fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) //
2,
(1024 + textsize[1]) //
2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
elif valid_hand(hand_confidence_avg, gesture) and gesture == 4:
print('RAISE HAND PROB : ', prob)
'''Counter'''
if prev_state is None:
prev_state = ('raise_hand', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 5.5:
prev_state = ('raise_hand', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'raise_hand':
prev_state = ('raise_hand', time.perf_counter())
# print(prev_state)
else:
# print(time.perf_counter() - prev_state[1])
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_raisehand, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
if time.perf_counter() - prev_state[1] > 0.5:
print_msg = True
bottomLeftCornerOfText = (450, 500)
fontColor = (0, 255, 255)
fontScale = 3
msg_on_screen = 'HAND RAISED'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale, fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
elif moved:
'''Counter'''
if prev_state is None:
prev_state = ('detect_move', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 1.5:
prev_state = ('detect_move', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'detect_move':
prev_state = ('detect_move', time.perf_counter())
# print(prev_state)
else:
# print(msg_state)
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_movement, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
print_msg = True
bottomLeftCornerOfText = (150, 500)
fontColor = (255, 255, 255)
fontScale = 3
msg_on_screen = 'MOVEMENT DETECTED'
textsize = cv2.getTextSize(msg_on_screen, font, fontScale,
fontThickness)[0]
# print(textsize)
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
if print_msg:
# print(data.decode("utf-8"))
# bottomLeftCornerOfText = (550, 500)
# fontScale = 2
# fontColor = (255, 0, 0)
lineType = 2
cv2.rectangle(frame, (0, 0), (1280, 1024), fontColor, 40)
cv2.putText(frame, msg_on_screen, bottomLeftCornerOfText, font,
fontScale, fontColor, lineType)
cv2.imshow("Openpose 1.4.0 Webcam", frame) # datum.cvOutputData
# Always clean up
cam.release()
cv2.destroyAllWindows()
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
print(e)
sys.exit(-1)
'Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?'
)
raise e
cap = cv2.VideoCapture(0)
params = dict()
params["model_folder"] = "../../../models/"
params["hand"] = True
params["hand_detector"] = 2
params["body"] = 0
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
handRectangles = [
# Left/Right hands person 0
[
op.Rectangle(0., 0., 0., 0.),
op.Rectangle(160., 80., 320., 320.),
]
]
portx = "COM4"
#波特率，标准值之一：50,75,110,134,150,200,300,600,1200,1800,2400,4800,9600,19200,38400,57600,115200
bps = 9600
#超时设置,None：永远等待操作，0为立即返回请求结果，其他值为等待超时时间(单位为秒）
timex = 5
# 打开串口，并得到串口对象
ser = serial.Serial(portx, bps, timeout=timex)
pointpair = [[20, 19], [19, 18], [18, 17], [17, 0], [16, 15], [15, 14],
[14, 13], [13, 0], [12, 11], [11, 10], [10, 9], [9, 0], [8, 7],
[7, 6], [6, 5], [5, 0], [4, 3], [3, 2], [2, 1], [1, 0], [17, 13],
[13, 9], [9, 5], [5, 1]]
while True:
depth_intrin = aligned_depth_frame.profile.as_video_stream_profile().intrinsics
# color_intrin = color_frame.profile.as_video_stream_profile().intrinsics
# print(f'width: {color_intrin.width} height: {color_intrin.height}')
# depth_to_color_extrin = aligned_depth_frame.profile.get_extrinsics_to(color_frame.profile)
# default image from RS: w848 h480. seems like max resolu input for openpose: 300x300
r1x = 300
r1y = 100
r2w = 300 # 300
r2h = 300 # 300
handRectangles = [
# Left/Right hands person 0
[
op.Rectangle(0., 0., 0., 0.), # disable left hand
op.Rectangle(r1x, r1y, r2w, r2h),
],
]
# Create new datum
datum = op.Datum()
datum.cvInputData = color_image
datum.handRectangles = handRectangles # need to modify here with a hand locator.
# Process and display image
opWrapper.emplaceAndPop([datum])
# print("Left hand keypoints: \n" + str(datum.handKeypoints[0]))
# print("Right hand keypoints: \n" + str(datum.handKeypoints[1]))
frame = datum.cvOutputData
# Starting OpenPose
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
# Read image and face rectangle locations
#imageToProcess = cv2.imread(args[0].image_path)
cap = cv2.VideoCapture(args[0].image_path)
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('output.avi', fourcc, 20.0,
(frame_width, frame_height))
faceRectangles = [
op.Rectangle(330.119385, 277.532715, 48.717274, 48.717274),
op.Rectangle(24.036991, 267.918793, 65.175171, 65.175171),
op.Rectangle(151.803436, 32.477852, 108.295761, 108.295761),
]
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
# Create new datum
datum = op.Datum()
datum.cvInputData = frame
datum.faceRectangles = faceRectangles
# Process and display image
opWrapper.emplaceAndPop([datum])
def main():
try:
'''
Part to get openpose & pyopenpose from built openpose.
Be sure to build the openpose system first.
Feel free to change directories of built python packages.
'''
# Import Openpose (Windows/Ubuntu/OSX)
dir_path = os.path.dirname(os.path.realpath(__file__))
try:
# Windows Import
if platform == "win32":
# Change these variables to point to the correct folder (Release/x64 etc.)
sys.path.append(dir_path + '/../../python/openpose/Release')
os.environ['PATH'] = os.environ[
'PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;'
import pyopenpose as op
# from openpose import *
else:
# Change these variables to point to the correct folder (Release/x64 etc.)
sys.path.append('../../python')
# If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it.
# sys.path.append('/usr/local/python')
from openpose import pyopenpose as op
# from openpose import *
except ImportError as e:
print(
'Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?'
)
raise e
'''
Customizing part for Openpose Python Wrapper.
For the project, only enable body & hand.
'''
params = dict()
params["model_folder"] = "../../../models/"
params["hand"] = True
params["hand_detector"] = 2
params["body"] = 1
params["net_resolution"] = '320x192' #20*11
# params["face"] = True
# params["disable_blending"] = True
# params["fps_max"] = 5
handRectangles = [[
op.Rectangle(128, 0, 1024, 1024),
op.Rectangle(0., 0., 0., 0.)
]]
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
# Process Image
datum = op.Datum()
datum.handRectangles = handRectangles
cam = cv2.VideoCapture(0) # modify here for camera number
cam.set(3, 1280)
cam.set(4, 1024)
pair_poseKeypoints = [[], []]
input_hands = []
prev_state = None
'''
Loading Model for Recognizing Hand Gestures
Please refer to model.py for various models trained for the task.
Current implementaiton is based on cnn_resample_100.pt which shows best demo performance.
If you would like to use other models, be sure to modify the codes below properly.
'''
model = HandGestureNet(n_channels=42, n_classes=5)
model.load_state_dict(torch.load('cnn_resample_100.pt'))
# model.load_state_dict(torch.load('CNN_mid12.pt'))
model.to('cuda')
model.eval()
'''
Message state contains time of state changes.
Used to prevent burst of messages in short time to the Zoom chat.
'''
msg_state = ('not_sent', time.perf_counter())
'''
Starting the Project
Pose & Hand Gesture Estimation
'''
while (cv2.waitKey(1) != 27):
if msg_state[0] == 'sent':
if time.perf_counter() - msg_state[1] > 2.5:
msg_state = ('not_sent', time.perf_counter())
ret, frame = cam.read()
datum.cvInputData = frame
opWrapper.emplaceAndPop([datum])
frame = datum.cvOutputData
'''If Person not in Camera'''
if datum.poseKeypoints.shape == ():
if msg_state[0] == 'not_sent':
# print('WHY NOT WORKING')
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_absent, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 3
fontColor = (0, 0, 255)
lineType = 2
fontThickness = 2
msg_on_screen = 'ABSENT!'
textsize = cv2.getTextSize(msg_on_screen, font, fontScale,
fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
cv2.rectangle(frame, (0, 0), (1280, 1024), (0, 0, 255), 20)
cv2.putText(frame, 'Absent!', bottomLeftCornerOfText, font,
fontScale, fontColor, lineType)
cv2.imshow("Openpose 1.4.0 Webcam", frame)
continue
'''If there are more than one people on camera'''
if len(datum.poseKeypoints) > 1:
if prev_state is not None and prev_state[0] == 'multi_people':
if prev_state[1] > 2:
if msg_state[0] == 'not_sent':
# print('WHY NOT WORKING')
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_multi, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
font = cv2.FONT_HERSHEY_SIMPLEX
fontScale = 3
fontColor = (0, 0, 255)
lineType = 2
fontThickness = 2
msg_on_screen = 'MUTLIPLE PEOPLE!'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale, fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
cv2.rectangle(frame, (0, 0), (1280, 1024), (0, 0, 255),
20)
cv2.putText(frame, msg_on_screen,
bottomLeftCornerOfText, font, fontScale,
fontColor, lineType)
cv2.imshow("Openpose 1.4.0 Webcam", frame)
msg_state = ('sent', time.perf_counter())
continue
else:
prev_state = ('multi_people', time.perf_counter())
'''Evaluate Movement & Confidence'''
del pair_poseKeypoints[0]
pair_poseKeypoints.append(datum.poseKeypoints[0])
body_confidence_avg = avg_pose_confidence(datum.poseKeypoints[0])
# print(body_confidence_avg)
moved = metric(pair_poseKeypoints)
'''
Evaluate Hand Gesture
Please modify the below if you use other models for hand gestures.
Refer to model.py
'''
# if len(input_hands) == 12: # normalizev2.pt
# del input_hands[0]
# input_hands.append(datum.handKeypoints[0][0])
# # print(len(input_hands))
# prob, gesture = None, None
# hand_confidence_avg = avg_list_confidence(input_hands)
# # if len(input_hands) == 12 and avg >= 0.1:
# if len(input_hands) == 12:
# # print('Confidence : ', hand_confidence_avg)
# prob, gesture = get_hand_gesture(model, input_hands)
# # print(prob, gesture)
norm_hand = normalize(datum.handKeypoints[0][0])
# print(norm_hand.shape)
if len(input_hands) == 100:
for i in range(5):
del input_hands[0]
for i in range(5):
input_hands.append(norm_hand)
# print(len(input_hands))
prob, gesture = None, None
hand_confidence_avg = avg_list_confidence(input_hands)
# if len(input_hands) == 12 and avg >= 0.1:
if len(input_hands) == 100:
# print('Confidence : ', hand_confidence_avg)
# print(input_hands[0][:][:2])
inputs = torch.FloatTensor(input_hands).to('cuda')
inputs = inputs[:, :, :2]
# print(inputs.size())
# prob, gesture = get_hand_gesture_cnn(model, input_hands[:][:][:1])
prob, gesture = get_hand_gesture_cnn(model, inputs)
print(prob, gesture)
'''Output Recognition Results'''
print_msg = False
font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCornerOfText = None
fontColor = None
fontScale = 3
fontThickness = 2
msg_on_screen = None
'''Thumbs DOWN'''
if valid_hand(hand_confidence_avg, gesture) and gesture == 1:
print('THUMBS DOWN PROB : ', prob)
if prob > 0: # mitigated by using softmax
'''Counter'''
if prev_state is None:
prev_state = ('thumbs_down', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 5.5:
prev_state = ('thumbs_down', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'thumbs_down':
prev_state = ('thumbs_down', time.perf_counter())
# print(prev_state)
else:
# print(time.perf_counter() - prev_state[1])
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_thumbsdown, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
if time.perf_counter() - prev_state[1] > 0.5:
print_msg = True
# bottomLeftCornerOfText = (450, 500)
fontColor = (255, 0, 0)
fontScale = 3
msg_on_screen = 'THUMBS DOWN'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale,
fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) //
2,
(1024 + textsize[1]) //
2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
'''Thumbs UP'''
elif valid_hand(hand_confidence_avg, gesture) and gesture == 2:
print('THUMBS UP PROB : ', prob)
'''Counter'''
if prob > 0:
if prev_state is None:
prev_state = ('thumbs up', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 5.5:
prev_state = ('thumbs_up', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'thumbs_up':
prev_state = ('thumbs_up', time.perf_counter())
# print(prev_state)
else:
# print(time.perf_counter() - prev_state[1])
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_thumbsup, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
if time.perf_counter() - prev_state[1] > 0.5:
print_msg = True
# bottomLeftCornerOfText = (450, 500)
fontColor = (0, 255, 0)
fontScale = 3
msg_on_screen = 'THUMBS UP'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale,
fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) //
2,
(1024 + textsize[1]) //
2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
'''RAISED HAND'''
elif valid_hand(hand_confidence_avg, gesture) and gesture == 4:
print('RAISE HAND PROB : ', prob)
'''Counter'''
if prev_state is None:
prev_state = ('raise_hand', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 5.5:
prev_state = ('raise_hand', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'raise_hand':
prev_state = ('raise_hand', time.perf_counter())
# print(prev_state)
else:
# print(time.perf_counter() - prev_state[1])
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_raisehand, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
if time.perf_counter() - prev_state[1] > 0.5:
print_msg = True
bottomLeftCornerOfText = (450, 500)
fontColor = (0, 255, 255)
fontScale = 3
msg_on_screen = 'HAND RAISED'
textsize = cv2.getTextSize(msg_on_screen, font,
fontScale, fontThickness)[0]
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
'''
Movement Detected
Please refer to metric.py for detection metrics
'''
elif moved:
'''Counter'''
if prev_state is None:
prev_state = ('detect_move', time.perf_counter())
# print(prev_state)
elif prev_state[0] == 'rest':
if time.perf_counter() - prev_state[1] > 1.5:
prev_state = ('detect_move', time.perf_counter())
# print(prev_state)
elif prev_state[0] != 'detect_move':
prev_state = ('detect_move', time.perf_counter())
# print(prev_state)
else:
# print(msg_state)
if msg_state[0] == 'not_sent':
conn.request(
"POST",
"/v2/chat/users/[email protected]/messages",
payload_movement, headers)
#
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))
msg_state = ('sent', time.perf_counter())
print_msg = True
bottomLeftCornerOfText = (150, 500)
fontColor = (0, 0, 255)
fontScale = 3
msg_on_screen = 'MOVEMENT DETECTED'
textsize = cv2.getTextSize(msg_on_screen, font, fontScale,
fontThickness)[0]
# print(textsize)
bottomLeftCornerOfText = ((1280 - textsize[0]) // 2,
(1024 + textsize[1]) // 2)
# if time.perf_counter() - prev_state[1] > 3.5:
# prev_state = ('rest', time.perf_counter())
'''Print messages according to states above'''
if print_msg:
lineType = 2
cv2.rectangle(frame, (0, 0), (1280, 1024), fontColor, 40)
cv2.putText(frame, msg_on_screen, bottomLeftCornerOfText, font,
fontScale, fontColor, lineType)
cv2.imshow("Openpose 1.4.0 Webcam", frame) # datum.cvOutputData
# Always clean up
cam.release()
cv2.destroyAllWindows()
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
print(e)
sys.exit(-1)
# Construct it from system arguments
# op.init_argv(args[1])
# oppython = op.OpenposePython()
# Starting OpenPose
opWrapper = op.WrapperPython()
opWrapper.configure(params)
opWrapper.start()
# Read image and face rectangle locations
imageToProcess = cv2.imread(args[0].image_path)
handRectangles = [
# Left/Right hands person 0
[
op.Rectangle(320.035889, 377.675049, 69.300949, 69.300949),
op.Rectangle(0., 0., 0., 0.),
],
# Left/Right hands person 1
[
op.Rectangle(80.155792, 407.673492, 80.812706, 80.812706),
op.Rectangle(46.449715, 404.559753, 98.898178, 98.898178),
],
# Left/Right hands person 2
[
op.Rectangle(185.692673, 303.112244, 157.587555, 157.587555),
op.Rectangle(88.984360, 268.866547, 117.818230, 117.818230),
]
]
# Create new datum
hnData = pickle.load(fr)
print("images loading")
imagePaths = list(paths.list_images('./step1'))
print('imagePaths', imagePaths)
start = time.time()
for idx, imagePath in enumerate(imagePaths):
classname = imagePath.split(os.path.sep)[-2]
filename = imagePath.split(os.path.sep)[-1]
imageToProcess = imread(imagePath)
handRectangles = [
# 왼손, 오른손
[
op.Rectangle(hnData[filename][0] - 100.0,
hnData[filename][1] - 100.0, 200.0, 200.0),
op.Rectangle(hnData[filename][2] - 100.0,
hnData[filename][3] - 100.0, 200.0, 200.0),
],
]
# Create new datum
datum = op.Datum()
datum.cvInputData = imageToProcess
datum.handRectangles = handRectangles
# Process and display image
opWrapper.emplaceAndPop([datum])
#결과값을 numpy로 변환
a = datum.handKeypoints
