def __init__(self, graphics_scene, parent=None):
super(ESGraphicsViewController, self).__init__(parent)
self.logger = logging.getLogger("GraphicsView")
self.graphics_scene = graphics_scene
self._init_ui()
self.setScene(self.graphics_scene)
self.mode = Mode.NO_OP # no operation
self.zoom_in_factor = 1.1
self.zoom_clamp = True
self.zoom = 0
self.zoom_step = 1
self.zoom_range = [-20, 1]
self.drag_edge = None
self.drag_start_socket = None
# to deal with event observers
self.drag_enter_observers = Observable()
self.drop_observers = Observable()
self.block_selected_observers = Observable()
self.no_block_selected_observers = Observable()
self.invalid_edge_observers = Observable()
self.update()
self.show()
def __init__(self, block_controller, music_controller=None):
self.logger = logging.getLogger("Interaction Controller")
self.block_controller = block_controller
self.music_controller = music_controller
self.robot_controller = None
self.wakeup_thread = None
self.animation_thread = None
self.engagement_thread = None
self.face_tracker_thread = None
self.timer_helper = TimerHelper()
self.music_command = None
self.animations_lst = []
self.animation_time = 0
self.animation_counter = -1
self.robot_volume = 50
self.robot_ip = None
self.port = None
self.robot_name = None
self.engagement_counter = 0
self.is_ready_to_interact = False
self.current_interaction_block = None
self.previous_interaction_block = None
self.interaction_blocks = None
self.interaction_design = None
self.stop_playing = False
self.execution_result = None
self.has_finished_playing_observable = Observable()
def __init__(self, session):
self.logger = logging.getLogger("Speech Handler")
self.session = session
self.tts = self.session.service("ALTextToSpeech")
self.animated_speech = self.session.service("ALAnimatedSpeech")
self.memory = self.session.service("ALMemory")
self.speech_recognition = self.session.service("ALSpeechRecognition")
self.audio = session.service("ALAudioDevice")
self.keyword_observers = Observable()
self.speech_certainty = 0.4
self.voice_pitch = 100
self.voice_speed = 85
self.current_keywords = []
self.is_listening = False
self.start_time = time.time()
# add a listener to the keyword stream
self.keyword_listener = None
self.keyword_events(subscribe=True)
def __init__(self, block_controller, music_controller=None):
super().__init__()
self.logger = logging.getLogger("Interaction Controller")
self.block_controller = block_controller
self.music_controller = music_controller
self.db_stream_controller = DBStreamController()
self.timer_helper = TimerHelper()
self.music_command = None
self.animations_lst = []
self.animation_time = 0
self.animation_counter = -1
self.robot_volume = 50
self.robot_name = None
self.robot_realm = None
self.current_interaction_block = None
self.previous_interaction_block = None
self.interaction_module = None
self.comm_style = CommunicationStyle.UNDEFINED
self.is_interacting = False
self.face_size_dict = {}
self.execution_result = None
self.executed_blocks = []
self.tablet_input = None
self.has_finished_playing_observers = Observable()
self.threads = []
self.on_connected_observers = Observable()
self.start_listening_to_db_stream()
def __init__(self, session):
self.logger = logging.getLogger("TabletHandler")
self.session = session
# add a listener to the tablet input stream
self.session.subscribe(self.on_tablet_input,
"rom.optional.tablet_input.stream")
self.tablet_input_observers = Observable()
self.is_listening = False
def __init__(self):
super(Scene, self).__init__()
Observable.__init__(self)
self.logger = logging.getLogger("Scene")
self.blocks = []
self.edges = []
self.graphics_scene = ESGraphicsScene(self)
self.history = SceneHistoryController(self)
class EngagementHandler(object):
def __init__(self, session):
self.logger = logging.getLogger("EngagementHandler")
self.session = session
self.last_time_detected = 0 # log the time
self.notification_interval = 1 # seconds
self.detected_faces_lst = []
# observers
self.face_detected_observers = Observable()
# subscribe to face events
self.session.subscribe(self.on_face_detected, "rom.optional.face.stream")
@inlineCallbacks
def on_face_detected(self, frame):
self.logger.info("Face detected: {}".format(frame))
try:
if frame is None or len(frame) == 0:
yield sleep(1)
else:
detected_face = frame["data"]["body"]
# skip empty frames
if detected_face and len(detected_face) > 0:
# check face size
face_size = frame["data"]["body"][0][2]
self.detected_faces_lst.append(face_size)
# > x seconds: notify observers
detection_interval = time.time() - self.last_time_detected
if detection_interval >= self.notification_interval:
self.logger.info("Detected a face: {} | after {}s".format(frame["data"],
detection_interval))
self.last_time_detected = time.time()
face_size = max(self.detected_faces_lst)
self.detected_faces_lst = []
self.face_detected_observers.notify_all(face_size)
yield sleep(1)
else:
yield sleep(1)
except Exception as e:
self.logger.error("Error while receiving the detected face: {}".format(e))
yield sleep(1)
def face_detection(self, start=False):
# start/close the face stream
self.session.call("rom.optional.face.stream" if start else "rom.optional.face.close")
def face_tracker(self, start=False):
self.session.call("rom.optional.face.face_tracker", start=start)
def __init__(self, session):
self.logger = logging.getLogger("TabletHandler")
self.session = session
self.tablet_service = self.session.service("ALTabletService")
self.memory = self.session.service("ALMemory")
# TODO: add a listener to the tablet input stream
self.tablet_input_listener = self.memory.subscriber("TabletInput")
self.tablet_input_listener.signal.connect(self.on_tablet_input)
self.tablet_input_observers = Observable()
self.is_listening = False
def __init__(self, session):
self.logger = logging.getLogger("EngagementHandler")
self.session = session
self.last_time_detected = 0 # log the time
self.notification_interval = 1 # seconds
self.detected_faces_lst = []
# observers
self.face_detected_observers = Observable()
# subscribe to face events
self.session.subscribe(self.on_face_detected, "rom.optional.face.stream")
class ConnectionHandler(object):
def __init__(self):
self.logger = logging.getLogger("Connection Handler")
self.rie = None
self.session_observers = Observable()
self.session = None
@inlineCallbacks
def on_connect(self, session, details=None):
self.logger.debug("Created session: {}".format(session))
self.session = session
yield self.session_observers.notify_all(session)
def start_rie_session(self, robot_name=None, robot_realm=None):
try:
if robot_realm is None:
# get the realm from config
name_key = "pepper" if robot_name is None else robot_name.lower(
)
robot_realm = config_helper.get_robot_settings(
)["realm"][name_key]
self.logger.info("{} REALM: {}".format(robot_name, robot_realm))
self.rie = Component(transports=[{
'url': u"wss://wamp.robotsindeklas.nl",
'serializers': ['msgpack'],
'max_retries': 0
}],
realm=robot_realm)
self.logger.info("** {}".format(threading.current_thread().name))
self.rie.on_join(self.on_connect)
self.logger.info("Running the rie component")
run([self.rie])
except Exception as e:
self.logger.error("Unable to run the rie component | {}".format(e))
def stop_session(self):
try:
if self.session:
self.session.leave()
self.session_observers.notify_all(None)
self.logger.info("Closed the robot session.")
else:
self.logger.info("There is no active session.")
except Exception as e:
self.logger.error("Error while closing rie session: {}".format(e))
def __init__(self, scene, start_socket, end_socket=None, edge_type=EdgeType.BEZIER):
super(Edge, self).__init__()
Observable.__init__(self) # explicit call to second parent class
self.logger = logging.getLogger("Edge")
self.scene = scene
# init sockets
self.start_socket = start_socket
self.end_socket = end_socket
self.graphics_edge, self.__edge_type = (None,) * 2
self.edge_type = edge_type
# add edge to the scene
self.scene.add_edge(self)
def __init__(self, session):
self.logger = logging.getLogger("Speech Handler")
self.session = session
self.keyword_observers = Observable()
self.speech_certainty = 0.4
self.voice_pitch = 100
self.voice_speed = 85
self.current_keywords = []
self.is_listening = False
self.start_time = time.time()
# add a listener to the keyword stream
self.session.subscribe(self.on_keyword, "rom.optional.keyword.stream")
def __init__(self,
block,
index=0,
position=Position.BOTTOM_LEFT,
socket_type=SocketType.INPUT,
edge_limit=-1):
super(Socket, self).__init__()
Observable.__init__(self) # explicit call to second parent class
self.logger = logging.getLogger("Socket")
self.block = block
self.index = index
self.position = position
self.socket_type = socket_type
self.edge_limit = edge_limit
self.edges = []
self._init_graphics()
def __init__(self, block_controller, music_controller=None, parent=None):
self.logger = logging.getLogger("SimulationController")
self.block_controller = block_controller
self.music_controller = music_controller
self.user_input = None
self.interaction_log = None
self.simulation_dock_widget = None
self._init_dock_widget(parent)
self.user_turn = False
self.timer_helper = TimerHelper()
self.animations_lst = []
self.current_interaction_block = None
self.previous_interaction_block = None
self.connecting_edge = None
self.execution_result = None
self.stop_playing = False
self.finished_simulation_observable = Observable()
def __init__(self, session):
self.logger = logging.getLogger("EngagementHandler")
self.session = session
self.memory = self.session.service("ALMemory")
self.face_service = self.session.service("ALFaceDetection")
self.tracker = self.session.service("ALTracker")
self.tracker_face_size = 0.1
self.last_time_detected = 0 # log the time
self.notification_interval = 1 # seconds
# observers
self.face_detected_observers = Observable()
self.face_subscriber = None
# subscribe to face events
self.face_events(subscribe=True)
def add_data_observer(self, data_key, observer):
try:
if data_key not in self._db_change_observers_dict.keys():
self._db_change_observers_dict[data_key] = Observable()
self._db_change_observers_dict[data_key].add_observer(observer)
except Exception as e:
self.logger.error(
"Error while adding observer '{}' to '{}' | {}".format(
observer, data_key, e))
def __init__(self, block, parent=None, bg_color=None):
super(ESGraphicsBlock, self).__init__(parent=parent)
Observable.__init__(self)
self.block = block
self.logger = logging.getLogger("GraphicsBlock")
# properties
block_size_settings = config_helper.get_block_size_settings()
self.width = block_size_settings["width"]
self.height = block_size_settings["height"]
self.rounded_edge_size = block_size_settings["rounded_edge_size"]
self.title_height = block_size_settings["title_height"]
self._padding = 5.0
self.is_moved = False
# pen
block_colors = config_helper.get_colors()
self._default_pen = QPen(QColor("#{}".format(block_colors['pen']['default'])))
self._default_pen.setWidthF(2.0)
self._selected_pen = QPen(QColor("#{}".format(block_colors['pen']['selected'])))
self._selected_pen.setWidthF(block_size_settings["selected_pen_width"])
self._title_brush = QBrush(QColor("#{}".format(block_colors['brush']['block_title'])))
self._bg_brush = QBrush(QColor("#{}".format(
block_colors['brush']['block_bg'] if bg_color is None else bg_color)))
# set title
self.__title = ""
self._init_title()
# init sockets
self._init_sockets()
# init content
self._init_content()
# init ui
self._init_ui()
def __init__(self,
scene,
title="Start",
socket_types=[],
pos=[],
parent=None,
icon=None,
output_edges=1,
bg_color=None):
super(Block, self).__init__()
Observable.__init__(self) # explicit call to second parent class
self.logger = logging.getLogger("Block")
self.scene = scene
self.parent = parent # any container
self.graphics_block = None
self.bg_color = bg_color
self.icon = icon
self.title = title # is also the pattern name
self.inputs = []
self.outputs = []
self.socket_spacing = 22
self._init_ui(socket_types, pos, output_edges)
# add observers
self.editing_observers = Observable()
self.settings_observers = Observable()
# add editing/settings listeners
self.content.editing_icon.clicked.connect(
lambda: self.editing_observers.notify_all(event=self))
self.content.settings_icon.clicked.connect(
lambda: self.settings_observers.notify_all(event=self))
# add block to the scene
self.scene.add_block(self)
def __init__(self, scene, parent=None):
super(BlockManagerWidget, self).__init__(parent)
self.logger = logging.getLogger("BlockManagerWidget")
self._init_ui(scene)
# Observables
self.drag_enter_observers = Observable()
self.drop_observers = Observable()
self.block_selected_observers = Observable()
self.no_block_selected_observers = Observable()
self.invalid_edge_observers = Observable()
self.right_click_block_observers = Observable()
class BlockManagerWidget(QWidget):
def __init__(self, scene, parent=None):
super(BlockManagerWidget, self).__init__(parent)
self.logger = logging.getLogger("BlockManagerWidget")
self._init_ui(scene)
# Observables
self.drag_enter_observers = Observable()
self.drop_observers = Observable()
self.block_selected_observers = Observable()
self.no_block_selected_observers = Observable()
self.invalid_edge_observers = Observable()
self.right_click_block_observers = Observable()
def _init_ui(self, scene):
self.setGeometry(200, 200, 800, 600)
# layout
self.layout = QVBoxLayout()
self.layout.setContentsMargins(0, 0, 0, 0)
# Graphics scene
self.scene = scene
# Graphics View
self.blocks_view = ESGraphicsViewController(self.scene.graphics_scene, self)
# add observers
self.blocks_view.drag_enter_observers.add_observer(self.on_drag_enter)
self.blocks_view.drop_observers.add_observer(self.on_drop)
self.blocks_view.block_selected_observers.add_observer(self.on_block_selected)
self.blocks_view.no_block_selected_observers.add_observer(self.on_no_block_selected)
self.blocks_view.invalid_edge_observers.add_observer(self.on_invalid_edge)
# set layout
self.layout.addWidget(self.blocks_view)
self.setLayout(self.layout)
self.setWindowTitle("Block Manager")
self.update_widget()
self.show()
def update_widget(self):
self.update()
# self.blocks_view.update()
def contextMenuEvent(self, event):
item = self.get_item_at(event.pos())
if hasattr(item, "block"):
self.logger.debug("item has block attribute: {}".format(item))
# notify observers
self.right_click_block_observers.notify_all(event)
super(BlockManagerWidget, self).contextMenuEvent(event)
def get_scene_position(self, mouse_position):
try:
return self.blocks_view.mapToScene(mouse_position)
except Exception as e:
self.logger.error("Error while mapping mouse position to scene! {}".format(e))
def zoom_scene(self, val):
self.blocks_view.zoom_scene(delta_y=val)
def get_item_at(self, pos):
return self.blocks_view.itemAt(pos)
def delete_selected(self):
self.blocks_view.delete_selected()
def clean_up(self):
# called on exit
del self.scene
del self.blocks_view
###
# Event Listeners
###
def on_drag_enter(self, event=None):
self.drag_enter_observers.notify_all(event)
def on_drop(self, event=None):
self.drop_observers.notify_all(event)
def on_block_selected(self, event=None):
self.block_selected_observers.notify_all(event)
def on_no_block_selected(self, event=None):
self.no_block_selected_observers.notify_all(event)
def on_invalid_edge(self, event=None):
self.invalid_edge_observers.notify_all(event)
class InteractionController(QtCore.QObject):
face_detected_signal = Signal(float)
def __init__(self, block_controller, music_controller=None):
super().__init__()
self.logger = logging.getLogger("Interaction Controller")
self.block_controller = block_controller
self.music_controller = music_controller
self.db_stream_controller = DBStreamController()
self.timer_helper = TimerHelper()
self.music_command = None
self.animations_lst = []
self.animation_time = 0
self.animation_counter = -1
self.robot_volume = 50
self.robot_name = None
self.robot_realm = None
self.current_interaction_block = None
self.previous_interaction_block = None
self.interaction_module = None
self.comm_style = CommunicationStyle.UNDEFINED
self.is_interacting = False
self.face_size_dict = {}
self.execution_result = None
self.executed_blocks = []
self.tablet_input = None
self.has_finished_playing_observers = Observable()
self.threads = []
self.on_connected_observers = Observable()
self.start_listening_to_db_stream()
def connect_to_robot(self, robot_name=None, robot_realm=None, robot_ip=None, robot_port=None):
self.robot_name = robot_name
self.robot_realm = robot_realm
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="connectRobot",
data_dict={"connectRobot": {"robotName": self.robot_name,
"robotRealm": self.robot_realm,
"robotIP": robot_ip,
"robotPort": robot_port},
"timestamp": time.time()})
def on_connect(self, data_dict=None):
try:
self.logger.info("Connect data received: {}".format(data_dict))
if data_dict and data_dict["isConnected"] is True:
self.logger.debug("Connected...")
self.on_connected_observers.notify_all(True)
else:
self.logger.info("Robot is not connected!")
except Exception as e:
self.logger.error("Error while extracting isConnected: {} | {}".format(data_dict, e))
def disconnect(self):
try:
self.logger.info("Disconnecting...")
self.engagement(start=False)
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="disconnectRobot",
data_dict={"disconnectRobot": True, "timestamp": time.time()})
self.logger.info("Disconnection was successful.")
except Exception as e:
self.logger.error("Error while disconnecting: {}".format(e))
return True
def reset(self):
self.execution_result = ""
self.tablet_input = ""
self.current_interaction_block = None
self.previous_interaction_block = None
self.executed_blocks = []
def on_exit(self):
self.logger.info("Exiting...")
self.db_stream_controller.stop_db_stream()
def stop_all_threads(self):
# close all threads
for thread in self.threads:
try:
thread.stop_running()
thread.quit()
thread.wait()
except Exception as e:
self.logger.error("Error while stopping thread: {} | {}".format(thread, e))
continue
self.threads = []
def is_connected(self):
success = False
conn = None
try:
conn = self.db_stream_controller.find_one(self.db_stream_controller.robot_collection, "isConnected")
if conn and conn["isConnected"] is True:
success = True
except Exception as e:
self.logger.error("Error while extracting isConnected: {} | {}".format(conn, e))
finally:
return success
def start_listening_to_db_stream(self):
observers_dict = {
"isConnected": self.on_connect,
"isExecuted": self.on_block_executed,
"isDisengaged": self.on_disengaged,
"startInteraction": self.on_start_interaction,
"tabletInput": self.on_tablet_input,
"faceDetected": self.on_face_detected
}
self.db_stream_controller.start_db_stream(observers_dict=observers_dict,
db_collection=self.db_stream_controller.robot_collection,
target_thread="qt")
def on_block_executed(self, data_dict=None):
self.execution_result = ""
try:
if self.is_interacting is False:
self.logger.info("Not in interaction mode!")
return False
self.logger.info("isExecuted data received: {}".format(data_dict))
self.execution_result = data_dict["isExecuted"]["executionResult"]
self.execute_next_block()
except Exception as e:
self.logger.error("Error while extracting isExecuted block: {} | {}".format(data_dict, e))
def on_tablet_input(self, data_dict=None):
self.tablet_input = ""
self.execution_result = ""
try:
if self.is_interacting is False:
self.logger.info("Not in interaction mode!")
return False
self.logger.info("Tablet data received: {}".format(data_dict))
self.tablet_input = data_dict["tabletInput"]
self.execution_result = data_dict["tabletInput"]
self.execute_next_block()
except Exception as e:
self.logger.error("Error while extracting tablet data: {} | {}".format(data_dict, e))
def on_face_detected(self, data_dict=None):
try:
self.logger.info("Received a detected face: {}".format(data_dict))
self.face_detected_signal.emit(data_dict["faceDetected"])
if self.is_interacting:
self.face_size_dict["{}".format(time.time()).replace(".", "_")] = data_dict["faceDetected"]
except Exception as e:
self.logger.error("Error while extracting face size: {}".format(e))
def on_disengaged(self, data_dict=None):
try:
self.logger.info("Disengaged data received: {}".format(data_dict))
self.execution_result = ""
self.stop_playing()
except Exception as e:
self.logger.error("Error while extracting disengaged data: {} | {}".format(data_dict, e))
def update_speech_certainty(self, speech_certainty=40.0):
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="speechCertainty",
data_dict={"speechCertainty": speech_certainty, "timestamp": time.time()})
def update_db_data(self, data_key, data_value):
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key=data_key,
data_dict={data_key: data_value, "timestamp": time.time()})
def wakeup_robot(self):
success = False
try:
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="wakeUpRobot",
data_dict={"wakeUpRobot": True, "timestamp": time.time()})
success = True
except Exception as e:
self.logger.error("Error while waking up the robot! | {}".format(e))
finally:
return success
def rest_robot(self):
try:
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="restRobot",
data_dict={"restRobot": True, "timestamp": time.time()})
except Exception as e:
self.logger.error("Error while setting the robot posture to rest: {}".format(e))
# TOUCH
# ------
def enable_touch(self):
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="enableTouch",
data_dict={"enableTouch": True, "timestamp": time.time()})
# BEHAVIORS
# ---------
def animate(self, animation_name=None):
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="animateRobot",
data_dict={"animateRobot": {"animation": animation_name, "message": ""},
"timestamp": time.time()})
def animated_say(self, message=None, animation_name=None):
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="animateRobot",
data_dict={
"animateRobot": {"animation": animation_name, "message": message},
"timestamp": time.time()})
def customized_say(self, interaction_block):
if interaction_block is None:
return
interaction_block.interaction_start_time = time.time()
block_to_execute = interaction_block.clone()
if "{answer}" in block_to_execute.message and self.execution_result:
block_to_execute.message = block_to_execute.message.format(answer=self.execution_result.lower())
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="interactionBlock",
data_dict={"interactionBlock": block_to_execute.to_dict,
"timestamp": time.time()})
# SPEECH
# ------
def say(self, message=None):
to_say = "Hello!" if message is None else message
if message is None:
self.logger.info(to_say)
self.animated_say(message=to_say)
def on_start_interaction(self, data_dict=None):
if self.is_interacting:
self.logger.info("Interaction is in progress...")
return False
self.logger.info("Starting the interaction!")
self.start_playing(int_block=self.get_starting_block())
def start_playing(self, int_block):
if int_block is None:
self.is_interacting = False
self.logger.warning("Couldn't start the interaction!")
return False
self.is_interacting = True
self.previous_interaction_block = None
self.current_interaction_block = int_block
self.current_interaction_block.execution_mode = ExecutionMode.NEW
self.logger.debug("Started playing the blocks")
self.execute_next_block() # start interacting
def stop_playing(self):
# self.tablet_image(hide=True)
self.is_interacting = False
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="resumeEngagement",
data_dict={"resumeEngagement": True, "timestamp": time.time()})
self.has_finished_playing_observers.notify_all(True)
self.logger.info("Stopped interacting.")
def get_starting_block(self):
try:
# check if the scene contains a valid start block
block = self.block_controller.get_block(pattern="start")
if block is None:
self.logger.warning("The scene doesn't contain a starting block! "
"Please add a 'START' block to play the interaction!")
return None
self.logger.info("Found a starting block.")
return block.parent
except Exception as e:
self.logger.error("Error while getting the start block: {}".format(e))
return None
def execute_next_block(self):
try:
self.logger.info("Executing next block.\n")
# self.block_controller.clear_selection()
connecting_edge = None
# check for remaining actions; otherwise, continue
if self.current_interaction_block.action_command and \
self.current_interaction_block.execution_mode is ExecutionMode.EXECUTING:
self.execute_action_command()
else:
# check if the interaction has just started
if self.previous_interaction_block is None:
self.previous_interaction_block = self.current_interaction_block
else:
# get the next block to say
self.current_interaction_block, connecting_edge = self.get_next_interaction_block()
if self.verify_current_interaction_block() is False:
return False
# change the block status
self.current_interaction_block.execution_mode = ExecutionMode.EXECUTING
# update selection
# self.current_interaction_block.set_selected(True)
# if connecting_edge is not None:
# connecting_edge.set_selected(True)
# send a request to say the robot message
self.customized_say(interaction_block=self.current_interaction_block)
except Exception as e:
self.logger.error("Error while executing next block: {}".format(e))
def get_next_interaction_block(self):
if self.current_interaction_block is None:
return None
next_block = None
connecting_edge = None
self.logger.info("Getting the next interaction block...")
try:
# 1- Check for the design module and execute it
if self.current_interaction_block.design_module and \
self.current_interaction_block.execution_mode is ExecutionMode.EXECUTING:
self.execute_interaction_module()
# 2- When the interaction module exists, get the next block from it
if self.current_interaction_block.is_hidden and self.interaction_module:
connecting_edge = None
next_block = self.interaction_module.get_next_interaction_block(self.current_interaction_block,
self.execution_result)
# check if reached the end of the interaction_module
if next_block is None and self.previous_interaction_block:
self.current_interaction_block = self.interaction_module.origin_block
# 3- Otherwise, Get the next block from the current design
if next_block is None:
next_block, connecting_edge = self.current_interaction_block.get_next_interaction_block(
execution_result=self.execution_result)
# Change current block status
self.current_interaction_block.execution_mode = ExecutionMode.COMPLETED
self.current_interaction_block.execution_result = self.execution_result
self.current_interaction_block.interaction_end_time = time.time()
# update previous block
self.previous_interaction_block = self.current_interaction_block
self.executed_blocks.append(self.current_interaction_block)
return next_block, connecting_edge
except Exception as e:
self.logger.error("Error while getting the next block! {}".format(e))
return next_block, connecting_edge
def engagement(self, start=False):
"""
@param start = bool
"""
self.logger.info("Engagement called with start = {}".format(start))
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="startEngagement",
data_dict={"startEngagement": start, "timestamp": time.time()})
def verify_current_interaction_block(self):
# if there are no more blocks, stop interacting
if self.current_interaction_block is None:
# stop interacting
self.stop_playing()
return False
return True
def execute_interaction_module(self):
self.interaction_module = ModuleFactory.create_module(self.current_interaction_block.design_module,
self.current_interaction_block,
self.block_controller)
self.logger.info("Interaction module: {}".format(self.interaction_module))
if self.interaction_module is None:
return
next_b = self.interaction_module.execute_module()
self.update_communication_style()
if next_b is not None:
next_b.is_hidden = True # just in case!
self.current_interaction_block.execution_mode = ExecutionMode.COMPLETED
self.previous_interaction_block = self.current_interaction_block
self.current_interaction_block = next_b
def update_communication_style(self):
try:
if "person" in self.interaction_module.filename.lower():
self.comm_style = CommunicationStyle.PERSON_ORIENTED
elif "task" in self.interaction_module.filename.lower():
self.comm_style = CommunicationStyle.TASK_ORIENTED
except Exception as e:
self.logger.error("Error while setting the communication style: {}".format(e))
def execute_action_command(self):
# check for remaining actions
if self.current_interaction_block.has_action(action_type=ActionCommand.PLAY_MUSIC):
self.on_music_mode()
return True
elif self.current_interaction_block.has_action(action_type=ActionCommand.WAIT):
self.on_wait_mode()
return True
elif self.current_interaction_block.has_action(action_type=ActionCommand.GET_RESERVATIONS):
self.on_get_reservations()
return True
return False
def on_get_reservations(self):
get_reservations_command = self.current_interaction_block.action_command
if get_reservations_command is not None:
reservations = get_reservations_command.execute()
def on_wait_mode(self):
wait_time = 1 # 1s
try:
if self.current_interaction_block is not None:
self.current_interaction_block.execution_mode = ExecutionMode.COMPLETED
wait_command = self.current_interaction_block.action_command
if wait_command is not None:
wait_time = wait_command.wait_time
except Exception as e:
self.logger.error("Error while setting wait time! {}".format(e))
finally:
self.logger.debug("Waiting for {} s".format(wait_time))
QTimer.singleShot(wait_time * 1000, self.execute_next_block)
def on_music_mode(self):
if self.music_controller is None:
self.logger.debug("Music player is not connected! Will skip playing music.")
self.on_music_stop()
else:
self.current_interaction_block.action_command.music_controller = self.music_controller
success = self.current_interaction_block.action_command.execute()
if success is True:
self.logger.debug("Playing now: {}".format(self.current_interaction_block.action_command.track))
# TODO: specify wait time as track time when play_time is < 0
# use action play time
wait_time = self.current_interaction_block.action_command.play_time
if wait_time <= 0:
wait_time = 30 # wait for 30s then continue
anim_key = self.current_interaction_block.action_command.animations_key
if anim_key is None or anim_key == "":
QTimer.singleShot(int(wait_time) * 1000, self.on_music_stop)
else:
self.on_animation_mode(music_command=self.current_interaction_block.action_command,
animation_time=int(wait_time))
# QTimer.singleShot(wait_time * 1000, self.on_music_stop)
else:
self.logger.warning("Unable to play music! {}".format(self.music_controller.warning_message))
self.on_music_stop()
def on_animation_mode(self, music_command, animation_time=0):
self.music_command = music_command
self.animations_lst = config_helper.get_animations()[music_command.animations_key]
self.animation_time = animation_time
self.animation_counter = -1
self.timer_helper.start()
self.execute_next_animation()
def on_animation_completed(self, val=None):
QTimer.singleShot(3000, self.execute_next_animation)
def execute_next_animation(self):
if self.music_command is None or len(self.animations_lst) == 0:
QTimer.singleShot(1000, self.on_music_stop)
elif self.timer_helper.elapsed_time() <= self.animation_time - 4: # use 4s threshold
# repeat the animations if the counter reached the end of the lst
self.animation_counter += 1
if self.animation_counter >= len(self.animations_lst):
self.animation_counter = 0
animation, message = self.get_next_animation(self.animation_counter)
if message is None or message == "":
self.animate(animation_name=animation)
else:
self.animated_say(message=message,
animation_name=animation)
else:
remaining_time = self.animation_time - self.timer_helper.elapsed_time()
QTimer.singleShot(1000 if remaining_time < 0 else remaining_time * 1000, self.on_music_stop)
def get_next_animation(self, anim_index):
anim, msg = ("", "")
try:
animation_dict = self.animations_lst[anim_index]
if len(animation_dict) > 0:
anim = animation_dict.keys()[0]
msg = animation_dict[anim]
except Exception as e:
self.logger.error("Error while getting next animation! {}".format(e))
finally:
return anim, msg
def on_music_stop(self):
self.logger.debug("Finished playing music.")
try:
if self.current_interaction_block is not None:
self.current_interaction_block.execution_mode = ExecutionMode.COMPLETED
if self.music_controller is not None:
self.music_controller.pause()
except Exception as e:
self.logger.error("Error while stopping the music! {}".format(e))
finally:
self.execute_next_block()
# TABLET
# ------
def tablet_image(self, hide=False):
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="hideTabletImage",
data_dict={"hideTabletImage": True, "timestamp": time.time()})
# PROPERTIES
# ==========
@property
def is_interacting(self):
return self._is_interacting
@is_interacting.setter
def is_interacting(self, val=False):
self._is_interacting = val
self.db_stream_controller.update_one(self.db_stream_controller.interaction_collection,
data_key="isInteracting",
data_dict={"isInteracting": val, "timestamp": time.time()})
class SpeechHandler(object):
def __init__(self, session):
self.logger = logging.getLogger("Speech Handler")
self.session = session
self.tts = self.session.service("ALTextToSpeech")
self.animated_speech = self.session.service("ALAnimatedSpeech")
self.memory = self.session.service("ALMemory")
self.speech_recognition = self.session.service("ALSpeechRecognition")
self.audio = session.service("ALAudioDevice")
self.keyword_observers = Observable()
self.speech_certainty = 0.4
self.voice_pitch = 100
self.voice_speed = 85
self.current_keywords = []
self.is_listening = False
self.start_time = time.time()
# add a listener to the keyword stream
self.keyword_listener = None
self.keyword_events(subscribe=True)
# SUBSCRIBE
# =========
def keyword_events(self, subscribe=True):
try:
if subscribe:
self.keyword_listener = self.memory.subscriber(
"WordRecognized")
self.keyword_listener.signal.connect(self.on_keyword)
else:
self.keyword_listener = None
except Exception as e:
self.logger.error(
"Error while subscribing to recognized word event: {}".format(
e))
# KEYWORDS
# ========
def on_keyword(self, value=None):
self.logger.info("OnKeyword: value = {}".format(value))
if (not self.is_listening) and (value is None or len(value) == 0):
self.logger.info("No keywords!")
else:
try:
certainty = value[1]
if certainty >= self.speech_certainty:
keyword = value[0]
if keyword in self.current_keywords:
self.logger.info(
"Detected keyword is in the list: {}".format(
value))
self.is_listening = False
self.clear_keywords()
self.keyword_stream(start=False)
self.keyword_observers.notify_all(keyword)
else:
self.logger.info(
"Keyword received '{}' is not in the list {}".
format(keyword, self.current_keywords))
else:
self.logger.info(
"OnKeyword received with low certainty: {}".format(
value))
except Exception as e:
self.logger.error(
"Error while getting the received answer! | {}".format(e))
def add_keywords(self, keywords=None):
# add keywords = [array of strings] to listen to
if keywords:
try:
self.logger.info("Adding keywords: {}".format(keywords))
self.speech_recognition.pause(True)
self.speech_recognition.setVocabulary(keywords, False)
self.speech_recognition.pause(False)
except Exception as e:
self.logger.error(
"Error while adding keywords: {} | {}".format(keywords, e))
def remove_keywords(self, keywords=None):
if keywords is None:
keywords = []
try:
self.speech_recognition.pause(True)
# self.speech_recognition.setVocabulary([k for k in self.current_keywords if k not in keywords], False)
except Exception as e:
self.logger.error("Error while removing keywords: {} | {}".format(
keywords, e))
def clear_keywords(self):
self.current_keywords = []
# self.remove_keywords()
def keyword_stream(self, start=False):
# start/close the speech recognition engine
self.logger.info("Speech recognition engine {}.".format(
"is starting" if start else "is stopped"))
if start:
self.speech_recognition.subscribe("SpeechHandler")
else:
self.speech_recognition.unsubscribe("SpeechHandler")
# Listener
# =========
def on_speech_event(self,
event_name=None,
task_id=None,
subscriber_identifier=None):
self.logger.info("Speech Event: {} | {} | {}".format(
event_name, task_id, subscriber_identifier))
self.on_start_listening()
def on_start_listening(self, results=None):
self.is_listening = True
self.add_keywords(keywords=self.current_keywords)
self.keyword_stream(start=True)
# ======
# SPEECH
# ======
def say(self, message="Hi"):
text = "\\vct={}\\\\rspd={}\\{}".format(int(self.voice_pitch),
int(self.voice_speed), message)
self.tts.say(text)
def animated_say(self, message="", animation_name=None):
text = "\\vct={}\\\\rspd={}\\{}".format(int(self.voice_pitch),
int(self.voice_speed), message)
if animation_name is None:
self.animated_speech.say(
text, {"bodyLanguageMode": "contextual"}) # vs random
else:
self.animated_speech.say(
'^start({}) {} ^wait({})'.format(animation_name, text,
animation_name),
{"bodyLanguageMode": "contextual"})
def set_volume(self, level=50.0):
vol = int(level) if level > 1 else int(level * 100)
self.logger.info("Setting volume to: {}".format(vol))
self.audio.setOutputVolume(vol)
def set_language(self, name="English"):
self.tts.setLanguage(name)
self.speech_recognition.setLanguage(name)
# MEMORY
# ======
def insert(self, data_dict=None):
try:
for key in data_dict.keys():
self.memory.insertData(key, data_dict[key])
except Exception as e:
print("Error while inserting '{}' into memory: {}".format(
data_dict, e))
def raise_event(self, event_name, event_value):
self.logger.info("Raised event '{}' to load '{}'".format(
event_name, event_value))
self.memory.raiseEvent(event_name, event_value)
def get_speech_event(self):
return self.memory.subscriber("ALAnimatedSpeech/EndOfAnimatedSpeech")
# PROPERTIES
# ==========
@property
def speech_certainty(self):
return self.__speech_certainty
@speech_certainty.setter
def speech_certainty(self, val):
val = abs(float(val))
self.__speech_certainty = val if (0 <= val <= 1) else (val / 100.0)
# HELPER
def print_data(self, result):
self.logger.info('Result received: {}'.format(result))
class SpeechHandler(object):
def __init__(self, session):
self.logger = logging.getLogger("Speech Handler")
self.session = session
self.keyword_observers = Observable()
self.speech_certainty = 0.4
self.voice_pitch = 100
self.voice_speed = 85
self.current_keywords = []
self.is_listening = False
self.start_time = time.time()
# add a listener to the keyword stream
self.session.subscribe(self.on_keyword, "rom.optional.keyword.stream")
# KEYWORDS
# ========
@inlineCallbacks
def on_keyword(self, frame):
if (not self.is_listening) and (frame is None or len(frame) == 0):
self.logger.info("No frames!")
else:
try:
certainty = frame["data"]["body"]["certainty"]
if certainty >= self.speech_certainty:
keyword = frame["data"]["body"]["text"]
if keyword in self.current_keywords:
self.logger.info(
"Detected keyword is in the list: {}".format(
frame["data"]))
self.is_listening = False
yield self.clear_keywords()
yield self.keyword_stream(start=False)
self.keyword_observers.notify_all(keyword)
else:
self.logger.info(
"Keyword received '{}' is not in the list {}".
format(keyword, self.current_keywords))
else:
self.logger.info(
"OnKeyword received with low certainty: {}".format(
frame))
except Exception as e:
self.logger.error(
"Error while getting the received answer! | {}".format(e))
def add_keywords(self, keywords=None):
# add keywords = [array of strings] to listen to
self.session.call("rom.optional.keyword.add",
keywords=[] if keywords is None else keywords)
def remove_keywords(self, keywords=None):
self.session.call("rom.optional.keyword.remove",
keywords=[] if keywords is None else keywords)
def clear_keywords(self):
self.current_keywords = []
self.session.call("rom.optional.keyword.clear")
def keyword_stream(self, start=False):
# start/close the keyword stream
self.session.call("rom.optional.keyword.stream"
if start else "rom.optional.keyword.close")
# Listener
# =========
def on_start_listening(self, results=None):
self.is_listening = True
self.add_keywords(keywords=self.current_keywords)
self.keyword_stream(start=True)
# ======
# SPEECH
# ======
def say(self, message="Hi"):
text = "\\vct={}\\\\rspd={}\\{}".format(int(self.voice_pitch),
int(self.voice_speed), message)
return self.session.call("rom.optional.tts.say", text=text)
def animated_say(self, message="", animation_name=None):
text = "\\vct={}\\\\rspd={}\\{}".format(int(self.voice_pitch),
int(self.voice_speed), message)
return self.session.call("rom.optional.tts.animate", text=text)
def set_volume(self, level=50.0):
vol = int(level) if level > 1 else int(level * 100)
self.logger.info("Setting volume to: {}".format(vol))
self.session.call("rom.actuator.audio.volume", volume=vol)
def set_language(self, name="en"):
self.session.call("rom.optional.tts.language", lang=u"{}".format(name))
self.session.call("rie.dialogue.config.language",
lang=u"{}".format(name))
# MEMORY
# =======
def insert(self, data_dict=None):
try:
self.session.call("rom.optional.tts.insert_data",
data_dict={} if data_dict is None else data_dict)
except Exception as e:
self.logger.error(
"Error while inserting '{}' into memory: {}".format(
data_dict, e))
# PROPERTIES
# ==========
@property
def speech_certainty(self):
return self.__speech_certainty
@speech_certainty.setter
def speech_certainty(self, val):
val = abs(float(val))
self.__speech_certainty = val if (0 <= val <= 1) else (val / 100.0)
# HELPER
# ======
@inlineCallbacks
def print_data(self, result):
yield self.logger.info('Result received: {}'.format(result))
class TabletHandler(object):
def __init__(self, session):
self.logger = logging.getLogger("TabletHandler")
self.session = session
self.tablet_service = self.session.service("ALTabletService")
self.memory = self.session.service("ALMemory")
# TODO: add a listener to the tablet input stream
self.tablet_input_listener = self.memory.subscriber("TabletInput")
self.tablet_input_listener.signal.connect(self.on_tablet_input)
self.tablet_input_observers = Observable()
self.is_listening = False
def show_webview(self, url="https://www.google.com", hide=False):
try:
if hide is True:
self.tablet_service.hideWebview()
else:
# Enable tablet wifi and display the webpage
self.tablet_service.enableWifi()
self.tablet_service.showWebview(url)
except Exception as e:
self.logger.error(e)
def on_tablet_input(self, value=None):
self.logger.info("Tablet Input: {}".format(value))
if not self.is_listening:
self.logger.info("Not listening!")
elif not value:
self.logger.info("No input received!")
else:
try:
self.logger.info("Received Tablet Input: {}".format(value))
self.is_listening = False
self.input_stream(start=False)
tablet_input = value
if type(tablet_input) is bytes:
tablet_input = tablet_input.decode('utf-8')
url = TabletHandler.create_tablet_url(page_name="index")
self.show_webview(url=url)
self.tablet_input_observers.notify_all(tablet_input)
except Exception as e:
self.logger.error(
"Error while getting the received tablet input: {}".format(
e))
def input_stream(self, start=False):
self.logger.info("{} Tablet Input stream.".format(
"Starting" if start else "Closing"))
self.is_listening = start
# TODO: start/close the input stream
@staticmethod
def create_tablet_url(page_name="index", url_params=None):
tablet_settings = config_helper.get_tablet_settings()
url = "http://{}/{}{}".format(
tablet_settings["ip"],
tablet_settings["pages"]["{}".format(page_name)],
"" if url_params is None else url_params)
return url
def set_image(self, image_path="img/help_charger.png", hide=False):
# If hide is false, display a local image located in img folder in the root of the robot
tablet_settings = config_helper.get_tablet_settings()
full_path = "http://{}/{}".format(tablet_settings["ip"], image_path)
self.logger.info("Image path: {}".format(full_path))
try:
self.tablet_service.hideImage(
) if hide is True else self.tablet_service.showImageNoCache(
full_path)
except Exception as e:
self.logger.error("Error while setting tablet image: {}".format(e))
def configure_wifi(self, security="WPA2", ssid="", key=""):
try:
self.tablet_service.configureWifi(security, ssid, key)
self.logger.debug("Successfully configured the wifi.")
except Exception as e:
self.logger.error("Error while configuring the wifi! {}".format(e))
def __init__(self):
self.logger = logging.getLogger("Connection Handler")
self.rie = None
self.session_observers = Observable()
self.session = None
class ESGraphicsViewController(QGraphicsView):
def __init__(self, graphics_scene, parent=None):
super(ESGraphicsViewController, self).__init__(parent)
self.logger = logging.getLogger("GraphicsView")
self.graphics_scene = graphics_scene
self._init_ui()
self.setScene(self.graphics_scene)
self.mode = Mode.NO_OP # no operation
self.zoom_in_factor = 1.1
self.zoom_clamp = True
self.zoom = 0
self.zoom_step = 1
self.zoom_range = [-20, 1]
self.drag_edge = None
self.drag_start_socket = None
# to deal with event observers
self.drag_enter_observers = Observable()
self.drop_observers = Observable()
self.block_selected_observers = Observable()
self.no_block_selected_observers = Observable()
self.invalid_edge_observers = Observable()
self.update()
self.show()
def _init_ui(self):
self.setRenderHints(QPainter.Antialiasing
| QPainter.HighQualityAntialiasing
| QPainter.TextAntialiasing
| QPainter.SmoothPixmapTransform)
self.setViewportUpdateMode(QGraphicsView.FullViewportUpdate)
# self.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse)
self.setDragMode(QGraphicsView.RubberBandDrag)
# enable drop
self.setAcceptDrops(True)
def keyPressEvent(self, event):
if event.key() in (Qt.Key_Delete, Qt.Key_Backspace):
self.delete_selected()
# elif event.key() == Qt.Key_S and event.modifiers() & Qt.ControlModifier:
# self.graphics_scene.scene.save_scene("graph.json")
# elif event.key() == Qt.Key_L and event.modifiers() & Qt.ControlModifier:
# self.graphics_scene.scene.load_scene("graph.json")
# elif event.key() == Qt.Key_Z \
# and event.modifiers() & Qt.ControlModifier \
# and not event.modifiers() & Qt.ShiftModifier:
# self.graphics_scene.scene.history.undo()
# elif event.key() == Qt.Key_Z \
# and event.modifiers() & Qt.ControlModifier \
# and event.modifiers() & Qt.ShiftModifier:
# self.graphics_scene.scene.history.redo()
elif event.key() == Qt.Key_H:
to_log = "\nHISTORY:\ttotal = {} -- step = {}\n".format(
len(self.graphics_scene.scene.history.history_stack),
self.graphics_scene.scene.history.current_step)
index = 0
for item in self.graphics_scene.scene.history.history_stack:
to_log = "{}\t#{} -- {}\n".format(to_log, index,
item['description'])
index += 1
self.logger.debug(to_log)
super(ESGraphicsViewController, self).keyPressEvent(event)
def dragEnterEvent(self, event):
self.drag_enter_observers.notify_all(event)
def dropEvent(self, event):
self.drop_observers.notify_all(event)
def mouseMoveEvent(self, event):
if self.mode == Mode.DRAG_EDGE:
pos = self.mapToScene(event.pos())
self.drag_edge.update_destination(pos.x(), pos.y())
super(ESGraphicsViewController, self).mouseMoveEvent(event)
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.left_mouse_button_press(event)
elif event.button() == Qt.RightButton:
self.right_mouse_button_press(event)
elif event.button() == Qt.MiddleButton:
self.middle_mouse_button_press(event)
else:
super(ESGraphicsViewController, self).mousePressEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.left_mouse_button_release(event)
elif event.button() == Qt.RightButton:
self.right_mouse_button_release(event)
elif event.button() == Qt.MiddleButton:
self.middle_mouse_button_release(event)
else:
super(ESGraphicsViewController, self).mouseReleaseEvent(event)
def left_mouse_button_press(self, event):
item = self.get_clicked_item(event)
# self.logger.debug("Item {} is clicked.".format(item))
if hasattr(item, "block"):
self.logger.debug("Selected item block: {}".format(item.block))
# send the item not the event
self.block_selected_observers.notify_all(item.block)
else:
self.no_block_selected_observers.notify_all(event)
# if item is None: # drag the scene
# pass # self.setDragMode(QGraphicsView.ScrollHandDrag)
if type(item) is ESGraphicsSocket:
if self.mode == Mode.NO_OP:
self.edge_drag_start(item)
return
if self.mode == Mode.DRAG_EDGE:
success = self.edge_drag_end(item)
if success:
return
super(ESGraphicsViewController, self).mousePressEvent(event)
def left_mouse_button_release(self, event):
item = self.get_clicked_item(event)
# if item is None:
# pass # self.setDragMode(QGraphicsView.NoDrag)
if self.mode == Mode.DRAG_EDGE:
# bypass the first click-release on the same socket
if type(item) is ESGraphicsSocket:
if item.socket != self.drag_start_socket:
success = self.edge_drag_end(item)
if success:
return
# if item is not None and self.dragMode() == QGraphicsView.RubberBandDrag and self.mode is Mode.NO_OP:
# print("selection changed")
# self.graphics_scene.scene.history.store("Selection changed")
super(ESGraphicsViewController, self).mouseReleaseEvent(event)
def right_mouse_button_press(self, event):
super(ESGraphicsViewController, self).mousePressEvent(event)
item = self.get_clicked_item(event)
if item is None:
# displays the number of blocks and edges in the scene
to_log = "\nSCENE:\n\tBlocks:"
for block in self.graphics_scene.scene.blocks:
to_log = "{}\n\t\t{}".format(to_log, block)
to_log = "{}\n\tEdges:".format(to_log)
for edge in self.graphics_scene.scene.edges:
to_log = "{}\n\t\t{}".format(to_log, edge)
self.logger.info(to_log)
elif isinstance(item, ESGraphicsSocket):
self.logger.info("{} has edges: {}".format(item.socket,
item.socket.edges))
elif isinstance(item, ESGraphicsEdge):
self.logger.debug("{} connecting {} & {}".format(
item.edge, item.edge.start_socket.block.title,
item.edge.end_socket.block.title))
self.logger.debug("Edge is in {}: {} | in {}: {}".format(
item.edge.start_socket, item.edge
in item.edge.start_socket.edges, item.edge.end_socket,
item.edge in item.edge.end_socket.edges))
def right_mouse_button_release(self, event):
super(ESGraphicsViewController, self).mouseReleaseEvent(event)
def middle_mouse_button_press(self, event):
super(ESGraphicsViewController, self).mousePressEvent(event)
def middle_mouse_button_release(self, event):
super(ESGraphicsViewController, self).mouseReleaseEvent(event)
def wheelEvent(self, event):
modifiers = QApplication.keyboardModifiers()
if modifiers == Qt.ControlModifier:
self.zoom_scene(delta_y=event.angleDelta().y())
else:
super(ESGraphicsViewController, self).wheelEvent(event)
def zoom_scene(self, delta_y):
# zoom factor
zoom_out_factor = 1 / self.zoom_in_factor
# compute zoom
if delta_y > 0:
zoom_factor = self.zoom_in_factor
self.zoom += self.zoom_step
else:
zoom_factor = zoom_out_factor
self.zoom -= self.zoom_step
clamped = False
if self.zoom < self.zoom_range[0]:
self.zoom, clamped = self.zoom_range[0], True
if self.zoom > self.zoom_range[1]:
self.zoom, clamped = self.zoom_range[1], True
# scene scale
if not clamped or self.zoom_clamp is False:
self.scale(zoom_factor, zoom_factor)
###
# Helper Methods
###
def delete_selected(self):
self.delete_selected_edges()
self.delete_selected_blocks()
self.graphics_scene.scene.store("Deleted selected items")
def delete_selected_blocks(self):
for item in self.graphics_scene.selectedItems():
if hasattr(item, "block"):
item.block.remove()
def delete_selected_edges(self):
for item in self.graphics_scene.selectedItems():
if isinstance(item, ESGraphicsEdge):
item.edge.remove()
def get_clicked_item(self, event):
pos = event.pos()
return self.itemAt(pos)
def edge_drag_start(self, item):
self.mode = Mode.DRAG_EDGE
self.drag_start_socket = item.socket
# create a new edge with dotted line
self.drag_edge = Edge(scene=self.graphics_scene.scene,
start_socket=item.socket,
end_socket=None,
edge_type=EdgeType.BEZIER)
def edge_drag_end(self, item):
# update mode
self.mode = Mode.NO_OP
success = False
try:
# remove dragged edge
self.drag_edge.remove()
self.drag_edge = None
if type(item) is ESGraphicsSocket:
# check if the connection is valid
if self.is_valid_connection(item.socket):
# create edge
Edge(scene=self.graphics_scene.scene,
start_socket=self.drag_start_socket,
end_socket=item.socket,
edge_type=EdgeType.BEZIER)
# store
self.graphics_scene.scene.store("New edge created")
success = True
except Exception as e:
self.logger.error("Error while ending drag! {}".format(e))
self.invalid_edge_observers.notify_all(
"Error while creating the edge!")
finally:
return success
def is_valid_connection(self, other_socket):
# check if it's the same socket
if other_socket == self.drag_start_socket:
return False
# connected sockets should have opposite types (input vs output)
if other_socket.socket_type == self.drag_start_socket.socket_type:
self.invalid_edge_observers.notify_all(
"* Cannot connect two sockets of the same type ({})".format(
self.drag_start_socket.socket_type.name))
return False
# check if one of the sockets is connected to the other's block
if self.drag_start_socket.is_connected_to_block(other_socket.block) \
or other_socket.is_connected_to_block(self.drag_start_socket.block):
return False
# check the number of allowed edges for the each block
if not (self.drag_start_socket.can_have_more_edges()
and other_socket.can_have_more_edges()):
self.invalid_edge_observers.notify_all(
"* The Edge cannot be created: "
"The output has reached the max number of allowed edges!")
return False
# connect to a block once
# if other_socket.block.is_connected_to(self.drag_start_socket.block):
# return False
# the connection is valid
return True
class Block(Serializable, Observable):
def __init__(self,
scene,
title="Start",
socket_types=[],
pos=[],
parent=None,
icon=None,
output_edges=1,
bg_color=None):
super(Block, self).__init__()
Observable.__init__(self) # explicit call to second parent class
self.logger = logging.getLogger("Block")
self.scene = scene
self.parent = parent # any container
self.graphics_block = None
self.bg_color = bg_color
self.icon = icon
self.title = title # is also the pattern name
self.inputs = []
self.outputs = []
self.socket_spacing = 22
self._init_ui(socket_types, pos, output_edges)
# add observers
self.editing_observers = Observable()
self.settings_observers = Observable()
# add editing/settings listeners
self.content.editing_icon.clicked.connect(
lambda: self.editing_observers.notify_all(event=self))
self.content.settings_icon.clicked.connect(
lambda: self.settings_observers.notify_all(event=self))
# add block to the scene
self.scene.add_block(self)
def _init_ui(self, socket_types, pos, output_edges):
self.content = ESBlockContentWidget(block=self)
self.graphics_block = ESGraphicsBlock(block=self,
bg_color=self.bg_color)
self._init_sockets(socket_types, output_edges)
if pos is not None and len(pos) == 2:
self.set_pos(*pos)
def _init_sockets(self, socket_types, output_edges):
in_counter = 0
out_counter = 0
for st in socket_types:
if st is SocketType.INPUT:
self.inputs.append(
Socket(block=self,
index=in_counter,
position=Position.BOTTOM_LEFT,
socket_type=SocketType.INPUT))
in_counter += 1
else:
self.outputs.append(
Socket(block=self,
index=out_counter,
position=Position.TOP_RIGHT,
socket_type=SocketType.OUTPUT,
edge_limit=output_edges))
out_counter += 1
def get_socket_position(self, index, position):
# set x
x, y = 0, 0 # for the left side
try:
if position in (Position.TOP_RIGHT, Position.BOTTOM_RIGHT,
Position.CENTER_RIGHT):
x = self.graphics_block.width
# set y
if position in (Position.CENTER_LEFT, Position.CENTER_RIGHT):
y = (self.graphics_block.height /
2) - index * self.socket_spacing
elif position in (Position.BOTTOM_LEFT, Position.BOTTOM_RIGHT):
# start on bottom
y = self.graphics_block.height - (
2 * self.graphics_block.rounded_edge_size
) - index * self.socket_spacing
else:
y = self.graphics_block.title_height + self.graphics_block.rounded_edge_size + index * self.socket_spacing
except Exception as e:
self.logger.debug(
"Error while getting the socket position! {}".format(e))
finally:
return [x, y]
def update_connected_edges(self):
for socket in self.inputs + self.outputs:
socket.update_edge_positions()
def is_connected_to(self, other_block):
"""
:param other_block:
:return: True if two blocks are connected; False otherwise.
"""
for edge in self.scene.edges:
if self in (edge.start_socket.block, edge.end_socket.block) and \
other_block in (edge.start_socket.block, edge.end_socket.block):
self.logger.info("{} is connected to {}".format(
self, other_block))
return True
return False
def get_edges(self, socket_type=SocketType.OUTPUT):
edges = []
for s in (self.outputs
if socket_type is SocketType.OUTPUT else self.inputs):
edges.extend([e for e in s.edges])
return edges
def get_connected_blocks(self, socket_type=SocketType.OUTPUT):
blocks = []
# go through target sockets and get the connected blocks
for output_socket in (self.outputs if socket_type is SocketType.OUTPUT
else self.inputs):
connected_sockets = output_socket.get_connected_sockets()
if connected_sockets is not None:
blocks.extend([s.block for s in connected_sockets])
self.logger.debug("# of Connected blocks: {}".format(
0 if blocks is None else len(blocks)))
return blocks
def remove(self):
# remove socket edges
for socket in (self.inputs + self.outputs):
# remove edges, if any
socket.disconnect_all_edges()
# remove block from scene
self.scene.remove_block(self)
self.graphics_block = None
def __str__(self):
return "<Block id {}..{}>".format((hex(id(self))[2:5]),
(hex(id(self))[-3:]))
def get_pos(self):
return self.graphics_block.pos() # QPointF
def set_pos(self, x, y):
self.graphics_block.setPos(x, y)
@property
def title(self):
return self.__title
@title.setter
def title(self, value):
self.__title = value
if self.graphics_block is not None:
self.graphics_block.title = self.title
@property
def icon(self):
return self.__icon
@icon.setter
def icon(self, value):
self.__icon = value
if self.graphics_block is not None:
self.graphics_block.set_title_pixmap(self.icon)
@property
def description(self):
return self.content.description
@description.setter
def description(self, desc):
self.content.description = desc
def set_selected(self, val):
if val is not None and self.graphics_block is not None:
self.graphics_block.setSelected(val)
@property
def pattern(self):
return self.parent.pattern if self.parent is not None else self.title
###
# SERIALIZATION
###
def serialize(self):
return OrderedDict([
("id", self.id), ("title", self.title), ("icon", self.icon),
("pos_x", self.graphics_block.scenePos().x()),
("pos_y", self.graphics_block.scenePos().y()),
("inputs", [s.serialize() for s in self.inputs]),
("outputs", [s.serialize() for s in self.outputs]),
("content", self.content.serialize()),
("parent", {} if self.parent is None else self.parent.serialize()),
("bg_color", self.bg_color)
])
def deserialize(self, data, hashmap={}):
self.id = data["id"]
hashmap[data["id"]] = self
self.icon = data["icon"]
self.title = data["title"]
self.set_pos(data["pos_x"], data["pos_y"])
self.bg_color = data["bg_color"] if "bg_color" in data.keys() else None
# set inputs and outputs
data["inputs"].sort(
key=lambda s: s["index"] + Position[s["position"]].value * 1000)
data["outputs"].sort(
key=lambda s: s["index"] + Position[s["position"]].value * 1000)
self.inputs = []
for s_data in data["inputs"]:
socket = Socket(block=self,
index=s_data["index"],
position=Position[s_data["position"]],
socket_type=SocketType[s_data["socket_type"]],
edge_limit=s_data["edge_limit"]
if "edge_limit" in s_data.keys() else 1)
socket.deserialize(s_data, hashmap)
self.inputs.append(socket)
self.outputs = []
for s_data in data["outputs"]:
socket = Socket(block=self,
index=s_data["index"],
position=Position[s_data["position"]],
socket_type=SocketType[s_data["socket_type"]],
edge_limit=s_data["edge_limit"]
if "edge_limit" in s_data.keys() else 1)
socket.deserialize(s_data, hashmap)
self.outputs.append(socket)
self.content.deserialize(data["content"], hashmap)
# set parent
if "parent" in data.keys() and len(data["parent"]) > 0:
self.parent = block_helper.create_block_parent(
data["parent"], hashmap)
return True
class SimulationController(object):
def __init__(self, block_controller, music_controller=None, parent=None):
self.logger = logging.getLogger("SimulationController")
self.block_controller = block_controller
self.music_controller = music_controller
self.user_input = None
self.interaction_log = None
self.simulation_dock_widget = None
self._init_dock_widget(parent)
self.user_turn = False
self.timer_helper = TimerHelper()
self.animations_lst = []
self.current_interaction_block = None
self.previous_interaction_block = None
self.connecting_edge = None
self.execution_result = None
self.stop_playing = False
self.finished_simulation_observable = Observable()
def _init_dock_widget(self, parent):
self.simulation_dock_widget = QtWidgets.QDockWidget(
"Simulation", parent)
size_policy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
size_policy.setHorizontalStretch(0)
size_policy.setVerticalStretch(0)
size_policy.setHeightForWidth(
self.simulation_dock_widget.sizePolicy().hasHeightForWidth())
self.simulation_dock_widget.setSizePolicy(size_policy)
self.simulation_dock_widget.setMinimumSize(QtCore.QSize(98, 150))
self.simulation_dock_widget.setFloating(False)
self.simulation_dock_widget.setFeatures(
QtWidgets.QDockWidget.AllDockWidgetFeatures)
self.simulation_dock_widget.setAllowedAreas(
QtCore.Qt.AllDockWidgetAreas)
self.simulation_dock_widget.setObjectName("simulationDockWidget")
# layout
widget_content = QtWidgets.QWidget()
widget_content.setObjectName("simulationDockWidgetContents")
grid_layout = QtWidgets.QGridLayout(widget_content)
grid_layout.setContentsMargins(11, 11, 11, 11)
grid_layout.setSpacing(6)
grid_layout.setObjectName("simulationGridLayout")
# display text edit
self.interaction_log = QtWidgets.QTextEdit(widget_content)
self.interaction_log.setAcceptDrops(False)
self.interaction_log.setAutoFillBackground(True)
self.interaction_log.setStyleSheet("background: white")
self.interaction_log.setUndoRedoEnabled(False)
self.interaction_log.setReadOnly(True)
self.interaction_log.setAcceptRichText(True)
self.interaction_log.setObjectName("simulationTextEdit")
grid_layout.addWidget(self.interaction_log, 0, 0, 1, 1)
# user input
self.user_input = QtWidgets.QLineEdit(widget_content)
self.user_input.setPlaceholderText("User Input")
self.user_input.returnPressed.connect(self.check_user_input)
self.user_input.setObjectName("simulationLineEdit")
grid_layout.addWidget(self.user_input, 1, 0, 1, 1)
widget_content.setLayout(grid_layout)
self.simulation_dock_widget.setWidget(widget_content)
def reset(self):
self.block_controller.clear_selection()
self.interaction_log.clear()
self.user_turn = False
self.current_interaction_block = None
self.previous_interaction_block = None
self.connecting_edge = None
self.execution_result = None
self.stop_playing = False
def start_simulation(self, int_block):
if int_block is None:
return
self.reset()
self.simulation_dock_widget.setFocus()
self.simulation_dock_widget.raise_()
self.current_interaction_block = int_block
self.execute_next_interaction_block()
def execute_next_interaction_block(self):
self.logger.debug("Getting the next interaction block...")
if self.current_interaction_block is None \
or self.user_turn is True:
return
self.logger.debug("Execution Result: {}".format(self.execution_result))
if self.previous_interaction_block is None: # simulation just started
# update previous block only
self.previous_interaction_block = self.current_interaction_block
else: # update previous and next blocks
self.previous_interaction_block = self.current_interaction_block
self.current_interaction_block, self.connecting_edge = \
self.current_interaction_block.get_next_interaction_block(execution_result=self.execution_result)
# reset execution_result
self.execution_result = None
self.simulate_interaction()
def simulate_interaction(self):
self.block_controller.clear_selection()
# if there are no more blocks, stop interacting
if self.current_interaction_block is None or self.stop_playing is True:
self.update_interaction_log(
robot_message="Finished the interaction!")
self.finished_simulation_observable.notify_all(True)
return True
else:
# execute the block
self.logger.debug("Executing: {}".format(
self.current_interaction_block.name))
self.current_interaction_block.set_selected(True)
if self.connecting_edge is not None:
self.connecting_edge.set_selected(True)
self.update_interaction_log(
robot_message=self.current_interaction_block.message)
if "question" in self.current_interaction_block.pattern.lower():
self.user_turn = True
return True
# check actions
if self.current_interaction_block.has_action(
action_type=ActionCommand.PLAY_MUSIC):
self.on_music_mode()
elif self.current_interaction_block.has_action(
action_type=ActionCommand.WAIT):
self.on_wait_mode()
else:
QTimer.singleShot(1500, self.execute_next_interaction_block)
def check_user_input(self):
user_input = self.user_input.text()
self.update_interaction_log(user_message=user_input)
# clear input
self.user_input.clear()
try:
# validate user input
if user_input.lower() == "exit":
# exit the interaction
self.user_turn = False
self.execution_result = user_input
self.update_interaction_log(robot_message="Ok, i'm exiting!")
QTimer.singleShot(1000, self.execute_next_interaction_block)
elif self.current_interaction_block.is_valid_user_input(
user_input=user_input) is True:
self.user_turn = False
self.execution_result = user_input
self.execute_next_interaction_block()
else:
self.update_interaction_log(
robot_message=
"Sorry, I didn't get your answer. Please try again!")
except Exception as e:
self.logger.error("Error while verifying user input! {}".format(e))
def on_wait_mode(self):
wait_command = self.current_interaction_block.action_command
if wait_command is None:
QTimer.singleShot(1000, self.execute_next_interaction_block)
else:
QTimer.singleShot(wait_command.wait_time * 1000,
self.execute_next_interaction_block)
def on_music_mode(self):
music_command = self.current_interaction_block.action_command
if self.music_controller is None or music_command is None:
QTimer.singleShot(1000, self.execute_next_interaction_block)
else:
music_command.music_controller = self.music_controller
success = music_command.execute()
if success is True:
message = "Playing now: {}".format(music_command.track)
self.update_interaction_log(robot_message=message)
# TODO: specify wait time as track time when play_time is < 0
# use action play time
wait_time = music_command.play_time
if wait_time <= 0:
wait_time = 30 # wait for 30s then continue
anim_key = music_command.animations_key
if anim_key is None or anim_key == "":
QTimer.singleShot(wait_time * 1000, self.on_music_stop)
else:
self.timer_helper.start()
self.animations_lst = config_helper.get_animations()[
music_command.animations_key]
self.on_animation_mode(music_command=music_command,
animation_time=wait_time,
counter=0)
else:
warning = "Unable to play music! {}".format(
self.music_controller.warning_message)
self.update_interaction_log(robot_message=warning)
QTimer.singleShot(2000, self.execute_next_interaction_block)
def on_animation_mode(self, music_command, animation_time=0, counter=0):
if music_command is None:
QTimer.singleShot(1000, self.on_music_stop)
if self.timer_helper.elapsed_time(
) <= animation_time - 4: # 4s threshold
anim_index = 0 if counter >= len(self.animations_lst) else counter
anim, msg = self.get_next_animation(anim_index)
robot_message = "{} - Executing {}".format(
"Animation time" if msg is None else msg, anim)
self.update_interaction_log(robot_message=robot_message)
QTimer.singleShot(
4000, lambda: self.on_animation_mode(
music_command, animation_time, anim_index + 1))
else:
remaining_time = animation_time - self.timer_helper.elapsed_time()
QTimer.singleShot(
1000 if remaining_time < 0 else remaining_time * 1000,
self.on_music_stop)
def get_next_animation(self, anim_index):
anim, msg = ("", "")
try:
animation_dict = self.animations_lst[anim_index]
if len(animation_dict) > 0:
anim = animation_dict.keys()[0]
msg = animation_dict[anim]
except Exception as e:
self.logger.error(
"Error while getting next animation! {}".format(e))
finally:
return anim, msg
def on_music_stop(self):
self.update_interaction_log(robot_message="Let' continue!")
self.music_controller.pause()
self.execute_next_interaction_block()
def update_interaction_log(self, robot_message=None, user_message=None):
if robot_message is not None and robot_message != "":
self._append_interaction_text(name="Robot",
color_name="green",
message=robot_message)
elif self.user_turn is True: # user_input is not None
# to prevent logging users' entered input when it's not their turn
self._append_interaction_text(name="User",
color_name="maroon",
message=user_message)
def _append_interaction_text(self, name, color_name, message):
self.interaction_log.setTextColor(QtGui.QColor(color_name))
self.interaction_log.append("{}:".format(name))
self.interaction_log.setTextColor(QtGui.QColor("black"))
self.interaction_log.append("{}".format(message))
self.logger.debug("{}: {}".format(name, message))
from es_common.model.interaction_block import InteractionBlock
from es_common.model.observable import Observable
block_observers = Observable()
def create_block_parent(parent_data, hashmap={}):
interaction_block = InteractionBlock.create_interaction_block(parent_data)
# deserialize
interaction_block.deserialize(parent_data, hashmap)
block_observers.notify_all(interaction_block.block)
return interaction_block
class EngagementHandler(object):
def __init__(self, session):
self.logger = logging.getLogger("EngagementHandler")
self.session = session
self.memory = self.session.service("ALMemory")
self.face_service = self.session.service("ALFaceDetection")
self.tracker = self.session.service("ALTracker")
self.tracker_face_size = 0.1
self.last_time_detected = 0 # log the time
self.notification_interval = 1 # seconds
# observers
self.face_detected_observers = Observable()
self.face_subscriber = None
# subscribe to face events
self.face_events(subscribe=True)
# SUBSCRIBE
# =========
def face_events(self, subscribe=True):
if subscribe:
self.face_subscriber = self.memory.subscriber("FaceDetected")
self.face_subscriber.signal.connect(self.on_face_detected)
self.face_service.subscribe("ESEngagementHandler")
else:
self.face_subscriber = None
self.face_service.unsubscribe("ESEngagementHandler")
def on_face_detected(self, value):
# self.logger.info("Face detected: {}".format(value))
try:
if value is None or value == []:
time.sleep(1)
else:
faces_info = value[1]
# skip empty frames
if faces_info and len(faces_info) > 0:
faces_info.pop() # rec info
face_size = 0.0
for f_info in faces_info:
tmp_size = max(f_info[0][3], f_info[0][4])
face_size = max(face_size, tmp_size)
# > x seconds: notify observers
detection_interval = time.time() - self.last_time_detected
if detection_interval >= self.notification_interval:
self.logger.info(
"Detected a face: {} | after {}s".format(
face_size, detection_interval))
self.last_time_detected = time.time()
self.face_detected_observers.notify_all(face_size)
else:
time.sleep(1)
except Exception as e:
self.logger.error(
"Error while receiving the detected face: {}".format(e))
def face_detection(self, start=False):
# start/close the face stream
if start:
self.face_service.subscribe("ESEngagementHandler")
else:
self.face_service.unsubscribe("ESEngagementHandler")
def face_tracker(self, start=False):
target_name = "Face"
if start is True:
self.tracker.registerTarget(target_name, self.tracker_face_size)
self.tracker.track(target_name)
else:
self.tracker.stopTracker()
self.tracker.unregisterAllTargets()