def __init__(self, model, height=256, device='CPU', openvino=False, tensorrt=False, extrinsics_path="./data/extrinsics.json", fx=1, canvas_shape=(720, 1280, 3)) -> None: if openvino: from modules.inference_engine_openvino import InferenceEngineOpenVINO self.net = InferenceEngineOpenVINO(model, device) else: from modules.inference_engine_pytorch import InferenceEnginePyTorch self.net = InferenceEnginePyTorch(model, device, tensorrt) try: with open(extrinsics_path, 'r') as f: self.extrinsics = json.load(f) except Exception: with open("./data/extrinsics.json", 'r') as f: self.extrinsics = json.load(f) traceback.print_exc() self.base_height = height self.fx = fx self.canvas_3d = np.zeros(canvas_shape, dtype=np.uint8) self.plotter = Plotter3d(self.canvas_3d.shape[:2]) # print("[INFO] plotter shape {}".format(self.plotter.shape)) print("[INFO] canvas shape {}".format(self.canvas_3d.shape))
def updatefig(i): print("updatefig",i) ax0.cla() ax1.cla() ax2.cla() imageIdx=(i+startFrom)*14 if (imageIdx >= (vidLength)-1): print('close') plt.close(fig) print("imageIdx",imageIdx) if (imageIdx == vidLength): print(f'imageIdx {imageIdx} == vidLength {vidLength}; closing!') plt.close(fig) frame = vid.get_data(imageIdx) ax1.imshow(frame) if withPose: poseIdx = imageIdx poses_3dFromImage=np.array([poseList[poseIdx][1:].reshape(19,3)]) # poses_3dFromImage=stand3dmatrix edgesFromImage = (Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3dFromImage.shape[0]).reshape((-1, 1, 1))).reshape((-1, 2)) canvas_3d = np.zeros((450, 450, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) plotter.plot(canvas_3d, poses_3dFromImage, edgesFromImage) ax0.imshow(canvas_3d) #imageIdx to csiIdx csiIndices=imageIdx2csiIndices(duration_in_sec,imageIdx,tsList,vidLength) if(len(csiIndices)>0): startCSIIdx=csiIndices[0] endCSIIdx=csiIndices[len(csiIndices)-1] print(startCSIIdx,'-',endCSIIdx) print(endCSIIdx-startCSIIdx+1) for j in range(0,64): if (6<=j<32 or 33<=j<59): textX=[] textY=[] for k in csiIndices: textX.append(tsList[k]/(10**6)) textY.append(csiList[k][j]) ax2.plot(textX,gaussian_filter(textY,sigma=1), label='CSI subcarrier') print("added") print('lastTS',tsList[endCSIIdx]) ax2.set_ylim([-10, +40]) # ax2.xlabel("Frame") # ax2.ylabel("Amplitude(dB)") return ax0,ax1,ax2
def updatefig(line): sLine = line % seqLen print("animate", sLine) if (sLine == seqLen - 1): fileIdx = 0 print("============filePath1", filePaths[fileIdx]) print("============color CSI", colorCSIs[fileIdx]) valueSTA = 0.1 valueAP = 0.1 try: curFileSTA = pd.read_csv(filePaths[fileIdx]) if (len(Channels) == 1): curFileSTA = curFileSTA[( curFileSTA['mac'] == my_filter_address)] curFileSTA = curFileSTA[(curFileSTA['len'] == 384)] curFileSTA = curFileSTA[(curFileSTA['stbc'] == 0)] curFileSTA = curFileSTA[(curFileSTA['rx_state'] == 0)] curFileSTA = curFileSTA[(curFileSTA['sig_mode'] == 1)] curFileSTA = curFileSTA[(curFileSTA['bandwidth'] == 1)] curFileSTA = curFileSTA[(curFileSTA['secondary_channel'] == 1)] tail = len(Channels) * 2 tail = 500 + 1 curCSI = curFileSTA['CSI_DATA'].tail(tail) RTcsiList = list(x for x in curCSI) curRSSI = curFileSTA['rssi'].tail(tail) RTrssiList = list(x for x in curRSSI) curTS = curFileSTA['local_timestamp'].tail(tail) tsSTAList = list(x for x in curTS) curChannel = curFileSTA['channel'].tail(tail) channelSTAList = list(x for x in curChannel) # print("timestamp",datetime.fromtimestamp(tsSTAList[0]+1611841591.5)) if (plotBySubcarrier): valueSTA = [] rssiSTA = [] tsSTA = [] startIndex = curRSSI.index[0] for i in range(len(RTcsiList) - 1): valueSTA.append(parseCSI(RTcsiList[i])) rssiSTA.append(RTrssiList[i]) tsSTA.append(tsSTAList[i]) except: print("catch animate RT") return [ax, ax0, ax1] # return [ax,ax0] if (shows[fileIdx] and isinstance(valueSTA, float) == False and len(valueSTA) > 0): csiAll = valueSTA tsAll = tsSTA rssiAll = rssiSTA print("last rssiAll", rssiAll[-1]) lastTS = tsAll[-1] / (10**6) lastTSbf = (tsAll[-1] / (10**6)) - (seqLen / 30) # print("last TS",lastTS) # print("last TSbf",lastTSbf) csiInRange = [] for i in range(len(csiAll)): if (tsAll[i] / (10**6) > lastTSbf): # if(True): csiInRange.append([tsAll[i] / (10**6)] + csiAll[i]) amplitudesAll = [ filterNullSC(rawCSItoAmp(csiInRange[j][1:])) for j in range(len(csiInRange)) ] csiIndices = [j for j in range(len(csiInRange))] poseIndices = [j for j in range((seqLen))] poseList = [] stepTS = (lastTS - lastTSbf) / seqLen for j in range(0, seqLen): poseList.append([lastTSbf + ((j + 1) * (stepTS))]) samplingedAmp, expectedTS = samplingCSI(csiInRange, csiIndices, poseList, poseIndices, paddingTo=seqLen) # Ploting Start for j in range(0, 52): textX = [] textY = [] for k in range(len(amplitudesAll)): curCsi = amplitudesAll[k][j] textX.append(csiInRange[k][0]) textY.append(curCsi) ax.plot(textX, gaussian_filter(textY, sigma=0), label='CSI subcarrier') textXSP = [] textYSP = [] for k in range(len(samplingedAmp)): curCsi = samplingedAmp[k][j] textXSP.append(expectedTS[k]) textYSP.append(curCsi) ax0.plot(textXSP, gaussian_filter(textYSP, sigma=0), label='CSI subcarrier') ax.set_xlim([textX[0], textX[-1]]) ax0.set_xlim([textXSP[0], textXSP[-1]]) sdSum = 0 for j in range(0, 52): subClist = [] for k in range(len(samplingedAmp)): subClist.append(samplingedAmp[k][j]) sdAmp = stdev(subClist) sdSum += sdAmp print("sum_diff", sdSum) if (sdSum > isActSDthreshold): # if(False): X = np.array([samplingedAmp]) X = featureEngineer(X) print(X.shape) global y_pred y_pred = model.predict(X) else: y_pred = False ax1.cla() if (isinstance(y_pred, bool) == False): print("have pose") if True: poses_3dFromImage = PAMtoPose(y_pred[0][sLine].reshape(3, 19, 19)) else: poses_3dFromImage = stand3dmatrix edgesFromImage = (Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3dFromImage.shape[0]).reshape( (-1, 1, 1))).reshape((-1, 2)) canvas_3d = np.zeros((450, 450, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) plotter.plot(canvas_3d, poses_3dFromImage, edgesFromImage) ax1.imshow(canvas_3d) # return [ax,ax0] return [ax, ax0, ax1]
args.add_argument('--no_show', help='Optional. Do not display output.', action='store_true') args.add_argument("-u", "--utilization_monitors", default='', type=str, help="Optional. List of monitors to show initially.") args = parser.parse_args() cap = open_images_capture(args.input, args.loop) stride = 8 inference_engine = InferenceEngine(args.model, args.device, stride) canvas_3d = np.zeros((720, 1280, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) canvas_3d_window_name = 'Canvas 3D' if not args.no_show: cv2.namedWindow(canvas_3d_window_name) cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback) file_path = args.extrinsics_path if file_path is None: file_path = Path(__file__).parent / 'data/extrinsics.json' with open(file_path, 'r') as f: extrinsics = json.load(f) R = np.array(extrinsics['R'], dtype=np.float32) t = np.array(extrinsics['t'], dtype=np.float32) is_video = cap.get_type() in ('VIDEO', 'CAMERA')
def main(): # model files check and download check_and_download_models(WEIGHT_PATH, MODEL_PATH, REMOTE_PATH) check_file_existance(FILE_PATH) # prepare input data canvas_3d = np.zeros((720, 1280, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) canvas_3d_window_name = 'Canvas3D' cv2.namedWindow(canvas_3d_window_name) cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback) with open(FILE_PATH, 'r') as f: extrinsics = json.load(f) R = np.array(extrinsics['R'], dtype=np.float32) t = np.array(extrinsics['t'], dtype=np.float32) if args.video is None: frame_provider = ImageReader([args.input]) is_video = False else: frame_provider = VideoReader(args.video) is_video = True fx = -1 delay = 1 esc_code = 27 p_code = 112 space_code = 32 mean_time = 0 img_mean = np.array([128, 128, 128], dtype=np.float32) base_width_calculated = False # net initialize env_id = ailia.get_gpu_environment_id() print(f'env_id: {env_id}') net = ailia.Net(MODEL_PATH, WEIGHT_PATH, env_id=env_id) # inference for frame_id, frame in enumerate(frame_provider): current_time = cv2.getTickCount() if frame is None: break if not base_width_calculated: IMAGE_WIDTH = frame.shape[1] * (IMAGE_HEIGHT / frame.shape[0]) IMAGE_WIDTH = int(IMAGE_WIDTH / STRIDE) * STRIDE net.set_input_shape((1, 3, IMAGE_HEIGHT, IMAGE_WIDTH)) base_width_calculated = True input_scale = IMAGE_HEIGHT / frame.shape[0] scaled_img = cv2.resize(frame, dsize=None, fx=input_scale, fy=input_scale) # better to pad, but cut out for demo scaled_img = scaled_img[:, 0:scaled_img.shape[1] - (scaled_img.shape[1] % STRIDE)] if fx < 0: # Focal length is unknown fx = np.float32(0.8 * frame.shape[1]) normalized_img = (scaled_img.astype(np.float32) - img_mean) / 255.0 normalized_img = np.expand_dims(normalized_img.transpose(2, 0, 1), axis=0) # exectution if is_video: input_blobs = net.get_input_blob_list() net.set_input_blob_data(normalized_img, input_blobs[0]) net.update() features, heatmaps, pafs = net.get_results() else: print('Start inference...') if args.benchmark: print('BENCHMARK mode') for i in range(5): start = int(round(time.time() * 1000)) features, heatmaps, pafs = net.predict([normalized_img]) end = int(round(time.time() * 1000)) print(f'\tailia processing time {end - start} ms') else: features, heatmaps, pafs = net.predict([normalized_img]) inference_result = (features[-1].squeeze(), heatmaps[-1].squeeze(), pafs[-1].squeeze()) poses_3d, poses_2d = parse_poses(inference_result, input_scale, STRIDE, fx, is_video) edges = [] if len(poses_3d): poses_3d = rotate_poses(poses_3d, R, t) poses_3d_copy = poses_3d.copy() x = poses_3d_copy[:, 0::4] y = poses_3d_copy[:, 1::4] z = poses_3d_copy[:, 2::4] poses_3d[:, 0::4], poses_3d[:, 1::4], poses_3d[:, 2::4] = -z, x, -y poses_3d = poses_3d.reshape(poses_3d.shape[0], 19, -1)[:, :, 0:3] edges = (Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3d.shape[0]).reshape( (-1, 1, 1))).reshape((-1, 2)) plotter.plot(canvas_3d, poses_3d, edges) if is_video: cv2.imshow(canvas_3d_window_name, canvas_3d) else: cv2.imwrite(f'Canvas3D_{frame_id}.png', canvas_3d) draw_poses(frame, poses_2d) current_time = (cv2.getTickCount() - current_time) / cv2.getTickFrequency() if mean_time == 0: mean_time = current_time else: mean_time = mean_time * 0.95 + current_time * 0.05 cv2.putText(frame, 'FPS: {}'.format(int(1 / mean_time * 10) / 10), (40, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) if is_video: cv2.imshow('ICV 3D Human Pose Estimation', frame) else: cv2.imwrite(args.savepath, frame) key = cv2.waitKey(delay) if key == esc_code: break if key == p_code: if delay == 1: delay = 0 else: delay = 1 if delay == 0 and args.rotate3d: key = 0 while (key != p_code and key != esc_code and key != space_code): plotter.plot(canvas_3d, poses_3d, edges) cv2.imshow(canvas_3d_window_name, canvas_3d) key = cv2.waitKey(33) if key == esc_code: break else: delay = 1 print('Script finished successfully.')
def updatefig(i): global y_pred global y_test global checkIndex # print("updatefig",i) i = i % seqLen if i == seqLen - 1: checkIndex = checkIndex + 1 if checkIndex == len(y_pred): checkIndex = 0 ax0.cla() ax1.cla() ax2.cla() poseIdx = i # if (poseIdx == len(csiList)-1): # print('close') # plt.close(fig) csiIdx = i # Plot 3D Pose poses_3dFromCSI = PAMtoPose(y_pred[checkIndex][poseIdx].reshape( 3, 19, 19)) edgesFromCSI = (Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3dFromCSI.shape[0]).reshape( (-1, 1, 1))).reshape((-1, 2)) canvas_3d = np.zeros((450, 450, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) plotter.plot(canvas_3d, poses_3dFromCSI, edgesFromCSI) ax0.imshow(canvas_3d) poses_3dFromImage = PAMtoPose(y_test[checkIndex][poseIdx].reshape( 3, 19, 19)) # poses_3dFromImage[0][9]=np.array([ 0,0,0]) # right shoulder # poses_3dFromImage[0][6]=np.array([ 0,0,0]) # left hip edgesFromImageGT = ( Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3dFromImage.shape[0]).reshape( (-1, 1, 1))).reshape((-1, 2)) canvas_3dGT = np.zeros((450, 450, 3), dtype=np.uint8) plotterGT = Plotter3d(canvas_3dGT.shape[:2]) plotterGT.plot(canvas_3dGT, poses_3dFromImage, edgesFromImageGT) ax1.imshow(canvas_3dGT) # Plot CSI for j in range(0, 52): textX = [] textY = [] for k in range(len(x_test[checkIndex])): curCsi = x_test[checkIndex][k] textX.append(k) textY.append(curCsi[j]) ax2.plot(textX, gaussian_filter(textY, sigma=0), label='CSI subcarrier') return ax0, ax1, ax2
def updatefig(i, noPlot=False): print("updatefig", i, label) # ax0.cla() # ax1.cla() # ax2.cla() imageIdx = i + startFrom print("imageIdx", imageIdx) if (noPlot == False and imageIdx == vidLength): print( f'imageIdx {imageIdx} == vidLength {vidLength}; closing!') plt.close(fig) try: frame = vid.get_data(imageIdx) input_scale = base_height / frame.shape[0] fx = np.float32(0.8 * frame.shape[1]) scaled_img = cv2.resize(frame, dsize=None, fx=input_scale, fy=input_scale) scaled_img = scaled_img[:, 0:scaled_img.shape[1] - ( scaled_img.shape[1] % stride)] # better to pad, but cut out for demo inference_result = net.infer(scaled_img) poses_3dFromImage, poses_2d = parse_poses( inference_result, input_scale, stride, fx, is_video) except: poses_3dFromImage = [] poses_2d = [] if len(poses_3dFromImage) == 0 or len( poses_2d) == 0 or poses_3dFromImage.all( ) == nullPose3D.all(): print("No pose detected ") # return False if noPlot==True else ax0,ax1,ax2 # return False poses_3dFromImage = np.array([np.zeros((19, 3))]) else: if True: poses_3dFromImage = rotate_poses(poses_3dFromImage, R, t) poses_3dFromImage = reshape_poses(poses_3dFromImage) else: poses_3dFromImage = stand3dmatrix if (noPlot == False): edgesFromImage = ( Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3dFromImage.shape[0]).reshape( (-1, 1, 1))).reshape((-1, 2)) canvas_3d = np.zeros((450, 450, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) plotter.plot(canvas_3d, poses_3dFromImage, edgesFromImage) ax0.imshow(canvas_3d) draw_poses(frame, poses_2d) ax1.imshow(frame) # Setting the values for all axes. csiIndices, parsedTimeInVid = imageIdx2csiIndicesPrecise( duration_in_sec, imageIdx, tsList, vidLength, lastsec) if (noPlot == True): print("parsedTimeInVid", parsedTimeInVid) parsedPoses_3dFromImage = np.array( poses_3dFromImage[0]).reshape(3 * 19) parsedTimeInVid_array = np.array([parsedTimeInVid]) pose3D_value.append( np.concatenate( (parsedTimeInVid_array, parsedPoses_3dFromImage))) if (len(csiIndices) > 0): startCSIIdx = csiIndices[0] endCSIIdx = csiIndices[len(csiIndices) - 1] print(startCSIIdx, '-', endCSIIdx) print(endCSIIdx - startCSIIdx + 1) for k in csiIndices: curParseCSI = parseCSI(csiList[k]) print("adding ", curParseCSI) if (curParseCSI != False): print("len check") print(k, len(curParseCSI), tsList[k]) if (len(curParseCSI) != 384): print("len not 384") continue print("isFloat check") isInt = True for l in range(384): if (isinstance(curParseCSI[l], int) == False): print(curParseCSI[l], " is not int") isInt = False break if isInt == False: continue csi_value.append([tsList[k]] + parseCSI(csiList[k])) print("added ", k) else: csi_value.append([tsList[k]] + [0 for l in range(384)]) print("added ", k, 'as 0s') else: for j in range(0, 64): if (6 <= j < 32 or 33 <= j < 59): textX = [] textY = [] for k in csiIndices: textX.append(tsList[k] / (10**6)) textY.append(rawCSItoAmp(parseCSI(x), 128)[k][j]) ax2.plot(textX, gaussian_filter(textY, sigma=1), label='CSI subcarrier') print("added") # print(tsList[csiIdx]) return False #if noPlot==True else ax0,ax1,ax2
def run_inference(args): from modules.inference_engine_pytorch import InferenceEnginePyTorch socket_server = SocketServer(args.port) joint_angle_calculator = JointAngleCalculator() stride = 8 model_path = os.path.join('models', 'human-pose-estimation-3d.pth') net = InferenceEnginePyTorch(model_path, "GPU") canvas_3d = np.zeros((720, 1280, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) canvas_3d_window_name = 'Canvas 3D' cv2.namedWindow(canvas_3d_window_name) cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback) file_path = None if file_path is None: file_path = os.path.join('data', 'extrinsics.json') with open(file_path, 'r') as f: extrinsics = json.load(f) R = np.array(extrinsics['R'], dtype=np.float32) t = np.array(extrinsics['t'], dtype=np.float32) frame_provider = ImageReader(args.images) is_video = False if args.video != '': frame_provider = VideoReader(args.video) is_video = True base_height = args.height_size fx = 1 # focal length delay = 1 esc_code = 27 p_code = 112 space_code = 32 mean_time = 0 for frame in frame_provider: current_time = cv2.getTickCount() if frame is None: break input_scale = base_height / frame.shape[0] scaled_img = cv2.resize(frame, dsize=None, fx=input_scale, fy=input_scale) scaled_img = scaled_img[:, 0:scaled_img.shape[1] - (scaled_img.shape[1] % stride)] # better to pad, but cut out for demo if fx < 0: # Focal length is unknown fx = np.float32(0.8 * frame.shape[1]) inference_result = net.infer(scaled_img) poses_3d, poses_2d = parse_poses(inference_result, input_scale, stride, fx, is_video) edges = [] if len(poses_3d): poses_3d = rotate_poses(poses_3d, R, t) poses_3d_copy = poses_3d.copy() x = poses_3d_copy[:, 0::4] y = poses_3d_copy[:, 1::4] z = poses_3d_copy[:, 2::4] poses_3d[:, 0::4], poses_3d[:, 1::4], poses_3d[:, 2::4] = -z, x, -y poses_3d = poses_3d.reshape(poses_3d.shape[0], 19, -1)[:, :, 0:3] edges = (Plotter3d.SKELETON_EDGES + 19 * np.arange(poses_3d.shape[0]).reshape((-1, 1, 1))).reshape((-1, 2)) plotter.plot(canvas_3d, poses_3d, edges) cv2.imshow(canvas_3d_window_name, canvas_3d) draw_poses(frame, poses_2d) current_time = (cv2.getTickCount() - current_time) / cv2.getTickFrequency() if mean_time == 0: mean_time = current_time else: mean_time = mean_time * 0.95 + current_time * 0.05 cv2.putText(frame, 'FPS: {}'.format(int(1 / mean_time * 10) / 10), (40, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) cv2.imshow('ICV 3D Human Pose Estimation', frame) key = cv2.waitKey(delay) if key == esc_code: break if key == p_code: if delay == 1: delay = 0 else: delay = 1 if delay == 0 or not is_video: # allow to rotate 3D canvas while on pause key = 0 while (key != p_code and key != esc_code and key != space_code): plotter.plot(canvas_3d, poses_3d, edges) cv2.imshow(canvas_3d_window_name, canvas_3d) key = cv2.waitKey(33) if key == esc_code: break else: delay = 1 joint_angles = joint_angle_calculator.calculate_angles(poses_3d) if joint_angles: socket_server.send_data(joint_angles)