def run_video(self):
cap = cv2.VideoCapture('Teach_action_new.mp4')
fps_time = 0
if cap.isOpened() is False:
print("Error opening video stream or file")
while cap.isOpened():
ret_val, image = cap.read()
humans = e.inference(image,
resize_to_default=True,
upsample_size=4.0)
key_points = TfPoseEstimator.get_keypoints(image, humans)
img = TfPoseEstimator.draw_one_human(image,
humans,
imgcopy=False,
score=0.8)
cv2.putText(img, "FPS: %f" % (1.0 / (time.time() - fps_time)),
(10, 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2)
cv2.imshow('video', image)
fps_time = time.time()
if cv2.waitKey(1) == 27:
break
def run_holo(self, window_name, player):
global global_action_p0, global_action_p1
global gamepoint_p0, gamepoint_p1
global p0_win_lose, p1_win_lose
global holo_action_p0, holo_action_p1
global status_data_p0, status_data_p1
global my_def_fail_p0, my_def_succ_p0, def_fail_p0, def_succ_p0
global my_def_fail_p1, my_def_succ_p1, def_fail_p1, def_succ_p1
global player_list, player_id_err_p0, player_id_err_p1
global frame_size_cp
global data_cp
global co_str_cp
print(player_list)
images = list()
final_action = 0
action_window = list() # action sliding window
for i in range(3): # sliding window size = 3
action_window.append(Skill.No_action.value)
fps_array = list() # FSP sliding window
for i in range(5): # sliding window size = 5
fps_array.append(15.0)
avg_fps = 15.0
step_size = 3
################### for debug ###################
# temp_cou = 0
# debug_root_path = 'debug/9_actions_11_class_MOD_4_0/'
# if not os.path.exists(debug_root_path):
# os.mkdir(debug_root_path)
# else:
# shutil.rmtree(debug_root_path)
# os.mkdir(debug_root_path)
# debug_file = open(debug_root_path + 'result.txt', 'w')
while (True):
no_human = 0 # no human flag
data = b''
try:
temp_data = self.socket.recv(4096)
frame_size = temp_data[:4]
frame_size_int = int.from_bytes(frame_size, byteorder='big')
#print(player + ' ' + str(frame_size_int))
temp_data = temp_data[4:]
data += temp_data
while (True):
if len(data) == frame_size_int + 256:
break
temp_data = self.socket.recv(4096)
data += temp_data
except ConnectionResetError: # 當 hololens 關閉時
self.run_holo_reset()
break
except ConnectionAbortedError: # 當 hololens 關閉時
self.run_holo_reset()
break
img_data = data[0:
frame_size_int] # 封包的前段是 HoloLens 傳過來的 image data
frame = np.fromstring(img_data, dtype=np.uint8)
dec_img = cv2.imdecode(frame, 1) # 解碼成可以讀取的影像檔
crop_img = dec_img.copy() # 複製一份給後面的crop用
if player == 'P0':
status_data_p0 = data[
frame_size_int:] # 封包的後段才是 HoloLens 傳過來的 status
elif player == 'P1': # status = "被對方的技能(ATK1)打到, 被對方的技能(ATK2)打到, 被對方的技能(ATK3)打到, 被對方的技能(ATK4)打到, end game, start new game"
status_data_p1 = data[frame_size_int:]
humans = e.inference(dec_img,
resize_to_default=True,
upsample_size=4.0)
img, key_points = TfPoseEstimator.draw_one_human(dec_img,
humans,
imgcopy=False,
score=0.8)
del fps_array[0]
fps_array.append(1.0 / (time.time() - self.fps_time))
avg_fps = sum(fps_array) / 5.0
if avg_fps > 16.0:
step_size = 3
elif avg_fps > 10.0:
step_size = 2
else:
step_size = 1
cv2.putText(img, "FPS: %f" % (avg_fps), (10, 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.imshow(window_name + player, img)
self.fps_time = time.time()
co_str = self.openpose_coordinate_to_str(key_points)
if key_points[1][0] == 0 and key_points[1][1] == 0 and key_points[
8][0] == 0 and key_points[8][1] == 0 and key_points[11][
0] == 0 and key_points[11][
1] == 0: # Neck, RHip, LHip == 0 => No human
x1 = 224 - 126
x2 = 224 + 126
no_human = 1
elif key_points[1][0] - 126 < 0:
x1 = 0
x2 = 252
elif key_points[1][0] + 126 > 447:
x1 = 196
x2 = 448
else:
x1 = key_points[1][0] - 126
x2 = key_points[1][0] + 126
crop_img = crop_img[:, x1:x2, :].copy()
#cv2.imshow('TSN', crop_img)
img_tsn = Image.fromarray(cv2.cvtColor(crop_img,
cv2.COLOR_BGR2RGB))
if self.count % step_size == 0 and len(images) == 2:
images.extend([img_tsn])
# print(images[0].size)
rst = trans(images)
rst = torch.unsqueeze(rst, 0)
my_output, my_pred = validate(model, rst) # predict action
self.action = int(my_pred[0][0])
top2 = int(my_pred[1][0])
top3 = int(my_pred[2][0])
top4 = int(my_pred[3][0])
top5 = int(my_pred[4][0])
################### for debug ###################
# if no_human == 0:
# detail = '{:d} ({:.2f}), {:d} ({:.2f}), {:d} ({:.2f}), {:d} ({:.2f}), {:d} ({:.2f})'.format(
# self.action, float(my_output[0][int(my_pred[0][0])]),
# top2, float(my_output[0][top2]),
# top3, float(my_output[0][top3]),
# top4, float(my_output[0][top4]),
# top5, float(my_output[0][top5])
# )
# img_dir = debug_root_path + '{:04d}.jpg'.format(temp_cou)
# cv2.imwrite(img_dir, crop_img, [int(cv2.IMWRITE_JPEG_QUALITY), 60])
# debug_file.write(detail + '\n')
# temp_cou += 1
if (self.action == 0):
self.action = num_class
del images[0]
# if no_human == 1: # 若其中一位玩家沒看著對方
# # holo_action_p0 = global_action_p0
# # holo_action_p1 = global_action_p1
# holo_action_p0 = Skill.No_action.value
# holo_action_p1 = Skill.No_action.value
# del action_window[0]
# action_window.append(Skill.No_action.value)
# elif (player == 'P0' and status_data_p0[10] == b'1'[0]): # HoloLens 端 P0 已經進入end game畫面
# holo_action_p1 = Skill.No_action.value # 遊戲 reset 階段辨識到的動作一率為 No action
# del action_window[0]
# action_window.append(Skill.No_action.value)
# elif (player == 'P1' and status_data_p1[10] == b'1'[0]): # HoloLens 端 P1 已經進入end game畫面
# holo_action_p0 = Skill.No_action.value # 遊戲 reset 階段辨識到的動作一率為 No action
# del action_window[0]
# action_window.append(Skill.No_action.value)
# else:
del action_window[0]
if (final_action == Skill.ATK_1_start.value
): # 如果上一個動作 是 螺旋丸的 start
if (self.action == Skill.ATK_1_start.value
or self.action == Skill.ATK_1_end.value
): # 如果現在的動作 是 螺旋丸的 start 或是 螺旋丸的 end
action_window.append(self.action)
elif (top2 == Skill.ATK_1_start.value
or top2 == Skill.ATK_1_end.value):
action_window.append(top2)
elif (top3 == Skill.ATK_1_start.value
or top3 == Skill.ATK_1_end.value):
action_window.append(top3)
else:
action_window.append(self.action)
elif (final_action == Skill.ATK_3_start.value
): # 如果上一個動作 是 歸派氣功的 start
if (self.action == Skill.ATK_3_start.value
or self.action == Skill.ATK_3_end.value
): # 如果現在的動作 是 歸派氣功的 start 或是 螺旋丸的 end
action_window.append(self.action)
elif (top2 == Skill.ATK_3_start.value
or top2 == Skill.ATK_3_end.value):
action_window.append(top2)
elif (top3 == Skill.ATK_3_start.value
or top3 == Skill.ATK_3_end.value):
action_window.append(top3)
else:
action_window.append(self.action)
else:
action_window.append(self.action)
temp = np.array([], np.int32)
for i in range(num_class +
1): # 因為 action 從 1 開始，而非 0，這樣後面取 argmax 才會正確
temp = np.append(temp, action_window.count(i))
final_action = np.argmax(temp)
if player == 'P0':
holo_action_p1 = final_action
elif player == 'P1':
holo_action_p0 = final_action
#print(action_window)
elif self.count % step_size == 0:
images.extend([img_tsn])
'''
if no_human == 1: # 若其中一位玩家沒看著對方
# holo_action_p0 = global_action_p0
# holo_action_p1 = global_action_p1
holo_action_p0 = Skill.No_action.value
holo_action_p1 = Skill.No_action.value
'''
# with open('Debug_for_multiplayer.txt', 'r') as f:
# debug_action = f.readline()
# holo_action_p0 = int(debug_action)
if (player == 'P0'):
if status_data_p0[10] == b'1'[
0] or gamepoint_p0 == 0.0: # HoloLens 端 P0 已經進入end game畫面
holo_action_p0 = Skill.No_action.value # 遊戲 reset 階段辨識到的動作一率為 No action
holo_action_p1 = Skill.No_action.value
if (player == 'P1'):
if status_data_p1[10] == b'1'[
0] or gamepoint_p1 == 0.0: # HoloLens 端 P1 已經進入end game畫面
holo_action_p0 = Skill.No_action.value # 遊戲 reset 階段辨識到的動作一率為 No action
holo_action_p1 = Skill.No_action.value
GameSystem(skill_damage=2.5, skill_wait_time=0.5,
player_=player) # 經過 GameSystem 判定雙方分數
co_str = co_str + str(self.count) + ',' + str(
holo_action_p0) + ',' + str(holo_action_p1)
co_str = co_str + ',' + str(gamepoint_p0) + ',' + str(
gamepoint_p1) + ',' + str(p0_win_lose) + ',' + str(p1_win_lose)
co_str = co_str + ',' + str(my_def_fail_p0) + ',' + str(
my_def_succ_p0) + ',' + str(def_fail_p0) + ',' + str(
def_succ_p0)
co_str = co_str + ',' + str(my_def_fail_p1) + ',' + str(
my_def_succ_p1) + ',' + str(def_fail_p1) + ',' + str(
def_succ_p1)
co_str = co_str + ',' + str(player_id_err_p0) + ',' + str(
player_id_err_p1)
###### Debug for 3D position estimation ######
# with open('Debug_for_3D_position_estimation.txt', 'r') as f:
# pre_z = f.readline()
# co_str = co_str + ',' + str(pre_z.strip())
#print(co_str)
co_str = bytes(co_str, 'ascii')
co_str = co_str + b',' + status_data_p0 + b',' + status_data_p1
try:
self.socket.send(co_str)
except ConnectionResetError: # 當 hololens 關閉時
self.run_holo_reset()
break
except ConnectionAbortedError: # 當 hololens 關閉時
self.run_holo_reset()
break
# ------------ reset data ------------ #
if (player == 'P0'):
my_def_fail_p0 = 0
my_def_succ_p0 = 0
def_fail_p0 = 0
def_succ_p0 = 0
if (status_data_p0[10] == b'1'[0] and len(player_list) == 1
): # 當player數只有一人(自己)，需要把對方的輸贏標記reset，不然等對手接上Server時會錯亂
p0_win_lose = 0
p1_win_lose = 0
gamepoint_p0 = 10.0
#print('one player reset(P0)')
elif (status_data_p0[10] == b'1'[0]
and status_data_p1[10] == b'1'[0]
and len(player_list) == 2):
p0_win_lose = 0
gamepoint_p0 = 10.0
print('---------- P0 Game system ready ----------')
if (player == 'P1'):
my_def_fail_p1 = 0
my_def_succ_p1 = 0
def_fail_p1 = 0
def_succ_p1 = 0
if (status_data_p1[10] == b'1'[0] and len(player_list) == 1):
p0_win_lose = 0
p1_win_lose = 0
gamepoint_p1 = 10.0
#print('one player reset(P1)')
elif (status_data_p0[10] == b'1'[0]
and status_data_p1[10] == b'1'[0]
and len(player_list) == 2):
p1_win_lose = 0
gamepoint_p1 = 10.0
print('---------- P1 Game system ready ----------')
# global_action_p0 = Skill.No_action.value
# global_action_p1 = Skill.No_action.value
####################### for unity_demo img_data #######################
# if player == 'P0':
# frame_size_cp = frame_size
# data_cp = img_data
# co_str_cp = co_str
self.count += 1
if cv2.waitKey(1) == 27:
break
cv2.destroyWindow(window_name + player)
print(datetime.datetime.now().strftime('%m/%d %H:%M:%S '), end='')
print('Client %s:%s disconnected.' % self.address, end='')
print('(' + player + ')')
#player_count -= 1
if (player == 'P0'):
player_list = player_list.replace('0', '')
if (player == 'P1'):
player_list = player_list.replace('1', '')
print(player_list)
self.socket.close()
def run_cam(self, cam_id):
global ori_RWrist, pre_RWrist, ori_RElbow, pre_RElbow
my_cam = cv2.VideoCapture(cam_id)
fps_time = 0
images = list()
while (True):
ret_val, image = my_cam.read()
kalmain_img = image.copy()
humans = e.inference(image,
resize_to_default=True,
upsample_size=4.0)
#key_points = TfPoseEstimator.get_keypoints(image, humans)
img, key_points = TfPoseEstimator.draw_one_human(image,
humans,
imgcopy=False,
score=0.8)
print(key_points)
if (key_points[3][0] != 0 or key_points[3][1] != 0):
ori_RElbow = np.array([[key_points[3][0]], [key_points[3][1]]],
np.float32)
kalman_RElbow.correct(ori_RElbow)
pre_RElbow = kalman_RElbow.predict()
if (key_points[4][0] != 0 or key_points[4][1] != 0):
ori_RWrist = np.array([[key_points[4][0]], [key_points[4][1]]],
np.float32)
kalman_RWrist.correct(ori_RWrist)
pre_RWrist = kalman_RWrist.predict()
print(pre_RElbow[0, 0])
print(pre_RElbow[1, 0])
cv2.circle(kalmain_img, (pre_RElbow[0, 0], pre_RElbow[1, 0]), 4,
(255, 0, 255), 5)
cv2.circle(kalmain_img, (pre_RWrist[0, 0], pre_RWrist[1, 0]), 4,
(255, 0, 0), 5)
cv2.line(kalmain_img, (pre_RElbow[0, 0], pre_RElbow[1, 0]),
(pre_RWrist[0, 0], pre_RWrist[1, 0]), (0, 0, 255), 2)
cv2.imshow('kalman', kalmain_img)
cv2.putText(img, "FPS: %f" % (1.0 / (time.time() - fps_time)),
(10, 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2)
cv2.imshow('camera' + str(cam_id), img)
'''
img = cv2.resize(img, (256, 256), interpolation=cv2.INTER_NEAREST)
img_tsn = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
if self.count % 3 == 0 and len(images) == 2:
images.extend([img_tsn])
# print(images[0].size)
# print(images[1].size)
# print(images[2].size)
rst = trans(images)
rst = torch.unsqueeze(rst, 0)
self.action = validate(model, rst)
#print(str(self.action) + ' ' + str(cam_id))
del images[0]
elif self.count % 3 == 0:
images.extend([img_tsn])
'''
fps_time = time.time()
self.count += 1
if cv2.waitKey(1) == 27:
break
)[-1] # 'D:/Dataset/Action/my_dataset/crop/1/1_0001/img_00001.jpg'
# print(img_name)
if int(img_dir.split('_')[-1].replace(
'/', '')) <= old_file_num[arr]: # 跳過舊的，只crop新的片段
break
if not os.path.exists(img_dir):
os.mkdir(img_dir)
img = cv2.imread(img_path)
image = img[:, 30:410, :].copy()
humans = e.inference(image,
resize_to_default=True,
upsample_size=4.0)
npimg, key_points = TfPoseEstimator.draw_one_human(image,
humans,
imgcopy=False,
score=0.8)
cv2.putText(npimg, "FPS: %f" % (1.0 / (time.time() - fps_time)),
(10, 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2)
cv2.imshow('tf-pose-estimation result', npimg)
fps_time = time.time()
# print(temp_class.Neck_x, temp_class.Neck_y)
if key_points[1][0] == 0 or key_points[1][1] == 0:
err += 1
print(img_name)
x1 = key_points[1][0] + 30 - 126
x2 = key_points[1][0] + 30 + 126
