def init_menu_widget(self): # Initialize the help tool tips self.left_swipe_tool_tip = WallframeImageWidget(1280, 400, self.assets_path + "/left.png") self.right_swipe_tool_tip = WallframeImageWidget(1280, 400, self.assets_path + "/right.png") self.click_tool_tip = WallframeImageWidget(1280, 400, self.assets_path + "/click.png") # add the tooltip widgets self.top_widget = QWidget() self.top_widget_layout = QHBoxLayout(self.top_widget) self.top_widget_layout.addWidget(self.left_swipe_tool_tip) self.top_widget_layout.addWidget(self.click_tool_tip) self.top_widget_layout.addWidget(self.right_swipe_tool_tip) self.top_widget_layout.setSpacing(0) self.top_widget.setFixedHeight(400) #self.top_widget.setStyleSheet("border-width: 0;") self.top_widget.setAutoFillBackground(True) p = self.top_widget.palette() p.setColor(self.top_widget.backgroundRole(), Qt.black) self.top_widget.setPalette(p) self.layout_.addWidget(self.top_widget) self.layout_.setSpacing(0) res_list = [item[1] + '/menu_icon.png' for item in self.app_paths_.items()] self.tileflow_widget = TileflowWidget(self, res_list) self.app_menu_items_ = [item[0] for item in self.app_paths_.items()] self.layout_.addWidget(self.tileflow_widget)
class AppMenu(WallframeAppWidget): Y_THRES = 50 X_LONG_THRES = 130 X_MID_THRES = 80 X_SHORT_THRES = 30 signal_hide_ = QtCore.Signal() signal_show_ = QtCore.Signal() signal_click_ = QtCore.Signal() # constructor def __init__(self): self.qt_app_ = QApplication(sys.argv) QWidget.__init__(self) # member variables self.current_users_ = [] # list of users self.users_ = {} # dict: (wallframe_id, user) self.num_users_ = 0 # total num of users self.focused_user_id_ = -1 # focused user self.app_menu_items_ = [] # list of the app names self.hidden_ = False self.run_ = False self.mouse_state = (0, 0) self.prev_user = None self.current_user = None self.state = "IDLE" #IDLE LEFT RIGHT # Indicates whether or not menu responds to user self.is_active = False # Currently running app or empty for menu self.current_app_id = "" self.setWindowFlags(QtCore.Qt.FramelessWindowHint) #self.setWindowFlags(self.windowFlags() | QtCore.Qt.WindowStaysOnTopHint) # ROS rospy.init_node('wallframe_app_tileflow_menu', anonymous=True) # ---- ROS subscriber --- self.user_state_sub_ = rospy.Subscriber("/wallframe/users/state", WallframeUserArray, self.user_state_cb) self.user_event_sub_ = rospy.Subscriber("/wallframe/users/events", WallframeUserEvent, self.user_event_cb) self.app_event_sub_ = rospy.Subscriber("/app/event", WallframeAppEvent, self.app_event_cb) self.toast_pub_ = rospy.Publisher("/wallframe/info/toast", String) self.app_request_pub_ = rospy.Publisher("/app/request", WallframeRequestApp) # ---- ROS get params ----- # height if rospy.has_param("/wallframe/core/params/height"): self.height_ = rospy.get_param("/wallframe/core/params/height") else: rospy.logerr("WallframeAppMenu: parameter [height] not found on server") # width if rospy.has_param("/wallframe/core/params/width"): self.width_ = rospy.get_param("/wallframe/core/params/width") else: rospy.logerr("WallframeAppMenu: parameter [width] not found on server") ### x ### if rospy.has_param("/wallframe/core/params/x"): self.x_ = rospy.get_param("/wallframe/core/params/x") else: rospy.logerr("WallframeAppMenu: parameter [x] not found on server") ### y ### if rospy.has_param("/wallframe/core/params/y"): self.y_ = rospy.get_param("/wallframe/core/params/y") else: rospy.logerr("WallframeAppMenu: parameter [y] not found on server") ### border scale ### if rospy.has_param("/wallframe/menu/params/border_scale"): self.border_scale_ = rospy.get_param("/wallframe/menu/params/border_scale") else: rospy.logerr("WallframeAppMenu: parameter [border_scale] not found on server") self.border_ = int(self.width_ * self.border_scale_) ### Cursor Icon ### if rospy.has_param("/wallframe/menu/params/cursor_path"): self.cursor_path_ = rospy.get_param("/wallframe/menu/params/cursor_path") else: rospy.logerr("WallframeAppMenu: parameter [cursor_path] not found on server") ### Cursor AltIcon ### if rospy.has_param("/wallframe/menu/params/cursor_path_alt"): self.cursor_path_alt_ = rospy.get_param("/wallframe/menu/params/cursor_path_alt") else: rospy.logerr("WallframeAppMenu: parameter [cursor_path_alt] not found on server") ### Application Locations ### if rospy.has_param("/wallframe/core/available_apps"): self.app_paths_ = rospy.get_param("/wallframe/core/available_apps") else: rospy.logerr("WallframeAppMenu: parameter [available_apps] not found on server") ### User Friendly Application names ### if rospy.has_param("/wallframe/core/app_ids"): self.app_ids_ = rospy.get_param("/wallframe/core/app_ids") else: rospy.logerr("WallframeAppMenu: parameter [app_ids] not found on server") ### get the assets path if rospy.has_param("/wallframe/core/tooltip/assets"): self.assets_path = rospy.get_param("/wallframe/core/tooltip/assets") else: rospy.logerr("WallframeAppMenu: parameter [assets] not found on server") ### Default App ### if rospy.has_param("/wallframe/core/default_app"): self.default_app_id_ = rospy.get_param("/wallframe/core/default_app") else: rospy.logerr("WallframeAppMenu: parameter [default_app] not found on server") ### Go to screensaver ### if rospy.has_param("/wallframe/menu/params/screensaver"): self.screensaver_ = rospy.get_param("/wallframe/menu/params/screensaver") else: rospy.logerr("WallframeAppMenu: parameter [screensaver] not found on server") if self.screensaver_ == True: rospy.logwarn("WallframeAppMenu: Configured to SHOW screensaver app when all users leave") else: rospy.logwarn("WallframeAppMenu: Configured to NOT default to screensaver app when all users leave") ### Workspace Limits ### if rospy.has_param("/wallframe/menu/params/workspace_size"): self.workspace_limits_ = rospy.get_param("/wallframe/menu/params/workspace_size") else: rospy.logerr("WallframeAppMenu: parameter [workspace_size] not found on server") ### Y Offset ### if rospy.has_param("/wallframe/menu/params/y_offset"): self.y_offset_ = rospy.get_param("/wallframe/menu/params/y_offset") else: rospy.logerr("WallframeAppMenu: parameter [y_offset] not found on server") ### Height Scaling ### if rospy.has_param("/wallframe/menu/params/height_percentage"): self.height_perc_ = rospy.get_param("/wallframe/menu/params/height_percentage") else: rospy.logerr("WallframeAppMenu: parameter [height_percentage] not found on server") #rospy.logwarn("WallframeAppMenu: height percentage set to " + str(self.height_perc_)) self.height_ = int(self.height_*self.height_perc_) self.setWindowTitle("Wallframe Main Menu") self.layout_ = QVBoxLayout() self.setWindowFlags(QtCore.Qt.FramelessWindowHint) self.show() self.resize(self.width_, self.height_) self.move(self.x_,self.y_) self.setLayout(self.layout_) # Init TileflowWidget self.init_menu_widget() # Timers self.ok_timer_ = QtCore.QTimer() self.connect(self.ok_timer_, QtCore.SIGNAL("timeout()"), self.check_ok) self.ok_timer_.start(15) self.run_ = True # Hide and Show Connections self.signal_show_.connect(self.show_menu) self.signal_hide_.connect(self.hide_menu) # Launch the screensaver self.load_app(self.default_app_id_) self.update_tooltip("tooltip_menu", "", "menu.png") def check_ok(self): if rospy.is_shutdown(): self.qt_app_.exit() else: self.update_tiles() pass pass #TODO this function should be in utilies def convert_workspace(self,user_pos): screen_pos = [] x_min = self.workspace_limits_[0] x_max = self.workspace_limits_[1] y_min = self.workspace_limits_[2] y_max = self.workspace_limits_[3] x_total = fabs(x_max)+fabs(x_min) y_total = fabs(y_max)+fabs(y_min) x_center = int(self.width_/2) y_center = int(self.height_/2) x_pos = int(x_center + (self.width_/x_total)*user_pos[0]) y_pos = int(y_center + (-(self.height_/y_total)*user_pos[1]) - self.y_offset_) if y_pos < 0.0: y_pos = 100 if y_pos > self.height_: y_pos = self.height_-100 if x_pos < 0.0: x_pos = 100 if x_pos > self.width_: x_pos = self.width_-100 screen_pos = [x_pos, y_pos] return screen_pos pass def init_menu_widget(self): # Initialize the help tool tips self.left_swipe_tool_tip = WallframeImageWidget(1280, 400, self.assets_path + "/left.png") self.right_swipe_tool_tip = WallframeImageWidget(1280, 400, self.assets_path + "/right.png") self.click_tool_tip = WallframeImageWidget(1280, 400, self.assets_path + "/click.png") # add the tooltip widgets self.top_widget = QWidget() self.top_widget_layout = QHBoxLayout(self.top_widget) self.top_widget_layout.addWidget(self.left_swipe_tool_tip) self.top_widget_layout.addWidget(self.click_tool_tip) self.top_widget_layout.addWidget(self.right_swipe_tool_tip) self.top_widget_layout.setSpacing(0) self.top_widget.setFixedHeight(400) #self.top_widget.setStyleSheet("border-width: 0;") self.top_widget.setAutoFillBackground(True) p = self.top_widget.palette() p.setColor(self.top_widget.backgroundRole(), Qt.black) self.top_widget.setPalette(p) self.layout_.addWidget(self.top_widget) self.layout_.setSpacing(0) res_list = [item[1] + '/menu_icon.png' for item in self.app_paths_.items()] self.tileflow_widget = TileflowWidget(self, res_list) self.app_menu_items_ = [item[0] for item in self.app_paths_.items()] self.layout_.addWidget(self.tileflow_widget) def send_app_request(self, command, app_id): #rospy.wait_for_service("/app/request") app_request = WallframeRequestApp() app_request.command = command app_request.app_id = app_id self.app_request_pub_.publish(app_request) def user_state_cb(self, msg): if self.run_: if self.focused_user_id_ != -1: self.prev_user = self.users_[self.focused_user_id_] current_users_ = msg.users self.num_users_ = len(current_users_) self.users_.clear() # To check if there is any focused user has_focused_user = False for user in current_users_: if user.focused == True: has_focused_user = True # if there is a new focused user then we need to delete the prev_user # data because it is for a diff user if(user.wallframe_id != self.focused_user_id_): self.prev_user = None self.focused_user_id_ = user.wallframe_id self.users_[user.wallframe_id] = user if(not has_focused_user): self.focused_user_id_ = -1 else: self.current_user = self.users_[self.focused_user_id_] def activate_menu(self): # show the menu self.signal_show_.emit() def deactivate_menu(self): # hide the menu self.signal_hide_.emit() def app_event_cb(self, msg): print "APP EVENT " + msg.app_id + " " + msg.status print "CURRENT APP " + self.current_app_id print "MENU ACTIVE " + str(self.is_active) if msg.status == "ready": if msg.app_id == self.current_app_id: self.deactivate_menu() self.update_tooltip("tooltip_menu", "", "menu.png") elif msg.status == "exit": if msg.app_id == self.default_app_id_: pass # TODO: Restart default app? if msg.app_id == self.current_app_id: self.toast_pub_.publish(String('App exited ' + self.app_ids_[msg.app_id])) self.is_active = True self.current_app_id = "" self.activate_menu() def terminate_current_app(self): if (self.current_app_id != self.default_app_id_ and self.current_app_id): self.toast_pub_.publish(String("Closing " + self.app_ids_[self.current_app_id])) self.terminate_app(self.current_app_id) self.current_app_id = "" def user_event_cb(self,msg): if self.run_: if msg.event_id == 'workspace_event': if msg.message == "all_users_left": # Go to screen saver self.terminate_current_app(); self.deactivate_menu() self.resume_app(self.default_app_id_) self.is_active = False elif msg.event_id == 'hand_event' and msg.user_id == self.focused_user_id_: if msg.message == 'hands_on_head': # Go to menu self.terminate_current_app() self.pause_app(self.default_app_id_); self.activate_menu() self.is_active = True elif msg.message == 'left_elbow_click' and self.is_active and not self.current_app_id: # Load the app triggering either a "ready" or "exit" event. self.is_active = False self.tileflow_widget.click() def clicked_on(self, ind): app_id = self.app_menu_items_[ind] self.load_app(app_id) def load_app(self, app_id): if (app_id != self.default_app_id_): self.update_tooltip("tooltip_menu", "", "loading.gif") self.toast_pub_.publish(String('Launching App ' + self.app_ids_[app_id])) self.current_app_id = app_id; self.send_app_request("launch", app_id) def terminate_app(self, app_id): self.send_app_request("terminate", app_id) def pause_app(self, app_id): self.send_app_request("pause", app_id) self.current_app_id = "" def resume_app(self, app_id): self.send_app_request("resume", app_id) self.current_app_id = app_id def update_tooltip(self, tooltip_name, text, background_path): rospy.wait_for_service(tooltip_name + '/update_tooltip') update_tooltip_cb = rospy.ServiceProxy(tooltip_name + '/update_tooltip', update_tooltip) try: success = update_tooltip_cb("Menu", text, background_path) except rospy.ServiceException as exc: rospy.logerr("WallframeTooltip: [tooltip " + tooltip_name + "] update_tooltip service could not update the text") def hide_tooltip_from_menu(self, tooltip_name): rospy.wait_for_service(tooltip_name + '/hide_tooltip') hide_cb = rospy.ServiceProxy(tooltip_name + '/hide_tooltip', hide_tooltip) try: success = hide_cb("Menu") except rospy.ServiceException as exc: rospy.logerr("WallframeTooltip: [tooltip " + tooltip_name + "] hide service could not hide the tooltip") def hide_menu(self): self.update_tooltip("tooltip_menu", "", "menu.png") self.hide() self.update() def show_menu(self): self.hide_tooltip_from_menu("tooltip_menu") self.show() self.update() def get_cursor_position_sensor(self): pos = self.users_[self.focused_user_id_].translations_mm[8] return (pos.x, pos.y) def get_cursor_position_mouse(self): pos = self.mouse_state return self.mouse_state def joint_position(self,user,name): return user.translations_body_mm[user.frame_names.index(name)] # Return whether or not there is a focused user with hands in cursor position. def user_has_cursor(self): if self.focused_user_id_ == -1: return False try: user = self.users_[self.focused_user_id_] right_hand = self.joint_position(user, 'left_hand') left_hand = self.joint_position(user, 'right_hand') head = self.joint_position(user, 'head') torso = self.joint_position(user, 'torso') midpoint = 0.75 * torso.y + 0.25 * head.y return right_hand.y > midpoint # The user id does not exist in the user map except IndexError as e: return False def check_gesture(self, prev_user, current_user): # LONG_RIGHT_SWIPE, SHORT_RIGHT_SWIPE, LONG_LEFT_SWIPE, SHORT_LEFT_SWIPE # old joint position values of the current user if prev_user.wallframe_id == current_user.wallframe_id: swipe_gesture = self.check_for_swipe(prev_user, current_user) if swipe_gesture: return swipe_gesture else: return None def check_for_right_swipe(self, prev_user, current_user): if self.joint_position(prev_user, 'left_hand').x > self.joint_position(prev_user, 'torso').x: dx = self.joint_position(current_user, 'left_hand').x - self.joint_position(prev_user, 'left_hand').x if self.state == "RIGHT": if -dx > self.X_SHORT_THRES: self.state = "IDLE" elif self.validate_y_for_swipe(self.joint_position(prev_user, 'left_hand').y, self.joint_position(current_user, 'left_hand').y, current_user): if dx > self.X_LONG_THRES: self.state = "RIGHT" return "LONG_RIGHT_SWIPE" elif dx > self.X_SHORT_THRES: self.state = "RIGHT" return "SHORT_RIGHT_SWIPE" return None def check_for_left_swipe(self, prev_user, current_user): if self.joint_position(prev_user, 'right_hand').x < self.joint_position(prev_user, 'torso').x: dx = self.joint_position(prev_user, 'right_hand').x - self.joint_position(current_user, 'right_hand').x if self.state == "LEFT": if -dx > self.X_SHORT_THRES: self.state = "IDLE" elif self.validate_y_for_swipe(self.joint_position(prev_user, 'right_hand').y, self.joint_position(current_user, 'right_hand').y, current_user): if dx > self.X_LONG_THRES: self.state = "LEFT" return "LONG_LEFT_SWIPE" elif dx > self.X_SHORT_THRES: self.state = "LEFT" return "SHORT_LEFT_SWIPE" return None def validate_y_for_swipe(self, prev_hand_y, current_hand_y,user): head = self.joint_position(user, 'head') torso = self.joint_position(user, 'torso') if current_hand_y >= torso.y and current_hand_y <= head.y and abs(current_hand_y - prev_hand_y) <= self.Y_THRES: return True else: return False def check_for_swipe(self, prev_user, current_user): #current_left_hand = self.joint_position(current_user, 'right_hand') #current_right_hand = self.joint_position(current_user, 'left_hand') #prev_left_hand = self.joint_position(prev_user, 'right_hand') #prev_right_hand = self.joint_position(prev_user, 'left_hand') # TODO think about what both the two swipe gestures happen together right_gesture = None left_gesture = None right_gesture = self.check_for_right_swipe(prev_user, current_user) left_gesture = self.check_for_left_swipe(prev_user, current_user) #if self.validate_y_for_swipe(prev_right_hand.y, current_right_hand.y,current_user): #right_gesture = self.check_for_right_swipe(prev_user, current_user) #if self.validate_y_for_swipe(prev_left_hand.y, current_left_hand.y,current_user): #left_gesture = self.check_for_left_swipe(prev_user, current_user) return left_gesture or right_gesture def update_tiles(self): if self.run_: if self.prev_user and self.current_user and self.is_active: gesture = self.check_for_swipe(self.prev_user,self.current_user) if gesture == "LONG_RIGHT_SWIPE": self.tileflow_widget.move_right(2) elif gesture == "SHORT_RIGHT_SWIPE": self.tileflow_widget.move_right(1) elif gesture == "LONG_LEFT_SWIPE": self.tileflow_widget.move_left(2) elif gesture == "SHORT_LEFT_SWIPE": self.tileflow_widget.move_left(1) # This is for testing with mouse def mouse_state_cb(self, msg): if self.run_: self.mouse_state = (msg.x, msg.y) def run(self): self.show() self.qt_app_.exec_() pass