def init(startMainLoop=True): global bus global session global ril0 global ril1 global power global network bus = SystemBus() # bus.subscribe(signal_fired=print) power = bus.get('.UPower') power.onPropertiesChanged = PropertyManager.propertiesChanged power = bus.get('.UPower', '/org/freedesktop/UPower/devices/battery_battery') power.onPropertiesChanged = PropertyManager.propertiesChanged LEDManager.ledsCharging(power.State == 1) ril0 = bus.get('org.ofono', '/ril_0') ril0.onCallAdded = PropertyManager.callStatusChanged ril0.onCallRemoved = PropertyManager.callStatusChanged ril1 = bus.get('org.ofono', '/ril_1') ril1.onCallAdded = PropertyManager.callStatusChanged ril1.onCallRemoved = PropertyManager.callStatusChanged network = bus.get('org.freedesktop.NetworkManager') network.onPropertiesChanged = PropertyManager.networkPropertiesChanged mtkCmd.WiFiStatusInfo(int(network.WirelessEnabled), 100) PropertyManager.init() session = SessionBus() session.subscribe(object='/com/canonical/pim/AddressBook', signal_fired=addressbookChanged) # help(ril0) # ril0['org.ofono.CallVolume'].onPropertyChanged = propertyChanged # ril0['org.ofono.VoiceCallManager'].onPropertyChanged = propertyChanged # print(dir(ril0['org.ofono.VoiceCallManager'])) # notifications = session.get('org.kde.kglobalaccel', '/component/kmix') # notifications.onglobalShortcutPressed = volumeChanged if startMainLoop: loop = GLib.MainLoop() loop.run()
def __init__(self, *args): _IdleObject.__init__(self) context = GObject.MainContext.default() self.bucket = bucket = Queue.Queue() # NOQA self.hello = None # with SessionBus() as bus: bus = SessionBus() ss = bus.get("org.scarlett", object_path='/org/scarlett/Listener') # NOQA time.sleep(1) ss_failed_signal = bus.subscribe(sender=None, iface="org.scarlett.Listener", signal="SttFailedSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=player_cb) ss_rdy_signal = bus.subscribe(sender=None, iface="org.scarlett.Listener", signal="ListenerReadySignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=player_cb) ss_kw_rec_signal = bus.subscribe(sender=None, iface="org.scarlett.Listener", signal="KeywordRecognizedSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=player_cb) ss_cmd_rec_signal = bus.subscribe(sender=None, iface="org.scarlett.Listener", signal="CommandRecognizedSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=command_cb) ss_cancel_signal = bus.subscribe(sender=None, iface="org.scarlett.Listener", signal="ListenerCancelSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=player_cb) pp.pprint((ss_failed_signal, ss_rdy_signal, ss_kw_rec_signal, ss_cmd_rec_signal, ss_cancel_signal)) logger.debug("ss_failed_signal: {}".format(ss_failed_signal)) logger.debug("ss_rdy_signal: {}".format(ss_rdy_signal)) logger.debug("ss_kw_rec_signal: {}".format(ss_kw_rec_signal)) logger.debug("ss_cmd_rec_signal: {}".format(ss_cmd_rec_signal)) logger.debug("ss_cancel_signal: {}".format(ss_cancel_signal)) ss.emitConnectedToListener('ScarlettTasker') loop.run() # THE ACTUAL THREAD BIT # self.manager = FooThreadManager(3) try: print("ScarlettTasker Thread Started") except Exception: ss_failed_signal.disconnect() ss_rdy_signal.disconnect() ss_kw_rec_signal.disconnect() ss_cmd_rec_signal.disconnect() ss_cancel_signal.disconnect() loop.quit() self.bucket.put(sys.exc_info()) raise
def run(self): self.log.info('DockedWatcher started') bus = SessionBus() bus.subscribe(None, None, "EventEmitted", "/com/ubuntu/Upstart", "drm-device-changed", 0, self.dbusSigHandler) self.gMainLoop.run()
def main_method(): # Call the 4 x method calls make_method_call_client_1() make_method_call_client_2() make_method_call_client_3() make_method_call_client_4() return True if __name__=="__main__": print("Starting...") # Create the dbus filter based on: org.example.demo.test dbus_filter = "/" + "/".join(BUS.split(".")) #print(dbus_filter) # Subscribe to dbus to monitor for server signal emissions # dbus_filter. E.g. /org/example/demo/test bus.subscribe(object = dbus_filter, signal_fired = cb_signal_emission) # Call function to make a method call GLib.timeout_add_seconds(interval=INTERVAL, function=main_method) loop.run() """ Notes: 1. Launch server_demo_6.py in one console 2. Then launch client_demo_6.py in another console. """
class ScarlettListenerI(threading.Thread, _IdleObject): """ Attempt to take out all Gstreamer logic and put it in a class ouside the dbus server. Cancellable thread which uses gobject signals to return information to the GUI. """ __gsignals__ = SCARLETT_LISTENER_I_SIGNALS device = PS_DEVICE hmm = HMM_PATH lm = LM_PATH dic = DICT_PATH # __dr = None __instance = None MAX_FAILURES = 10 def __init__(self, name, config_manager, *args): threading.Thread.__init__(self) _IdleObject.__init__(self) self.running = False self.finished = False self.ready_sem = threading.Semaphore(SEMAPHORE_NUM) self.queue = queue.Queue(QUEUE_SIZE) # Load in config object, and set default device information self._config_manager = config_manager self._graphviz_debug_dir = self._config_manager.cfg[ "graphviz_debug_dir"] self._device = self._config_manager.cfg["pocketsphinx"]["device"] self._hmm = self._config_manager.cfg["pocketsphinx"]["hmm"] self._lm = self._config_manager.cfg["pocketsphinx"]["lm"] self._dic = self._config_manager.cfg["pocketsphinx"]["dict"] self._fwdflat = bool( self._config_manager.cfg["pocketsphinx"]["fwdflat"]) self._bestpath = bool( self._config_manager.cfg["pocketsphinx"]["bestpath"]) self._dsratio = int( self._config_manager.cfg["pocketsphinx"]["dsratio"]) self._maxhmmpf = int( self._config_manager.cfg["pocketsphinx"]["maxhmmpf"]) self._bestpath = bool( self._config_manager.cfg["pocketsphinx"]["bestpath"]) self._silprob = float( self._config_manager.cfg["pocketsphinx"]["silprob"]) self._wip = float(self._config_manager.cfg["pocketsphinx"]["wip"]) # dotfile setup self._dotfile_listener = os.path.join(self._graphviz_debug_dir, "generator-listener.dot") self._pngfile_listener = os.path.join( self._graphviz_debug_dir, "generator-listener-pipeline.png") # self._handler = DbusSignalHandler() # Get a dbus proxy and check if theres a service registered called 'org.scarlett.Listener' # if not, then we can skip all further processing. (The scarlett-os-mpris-dbus seems not to be running) # self.__dr = DBusRunner.get_instance() logger.info("Initializing ScarlettListenerI") # This wil get filled with an exception if opening fails. self.read_exc = None self.dot_exc = None self.cancelled = False self.name = name self.setName("{}".format(self.name)) self.pipelines_stack = [] self.elements_stack = [] self.gst_bus_stack = [] self._message = "This is the ScarlettListenerI" # TODO: When we're ready to unit test, config this back in!!!!! # self.config = scarlett_config.Config() self.config = None self.override_parse = "" self.failed = 0 self.kw_found = 0 self.debug = False self.create_dot = True self.terminate = False self.capsfilter_queue_overrun_handler_id = None self._cancel_signal_callback = None # source: https://github.com/ljmljz/xpra/blob/b32f748e0c29cdbfab836b3901c1e318ea142b33/src/xpra/sound/sound_pipeline.py # NOQA self.bus = None self.bus_message_element_handler_id = None self.bus_message_error_handler_id = None self.bus_message_eos_handler_id = None self.bus_message_state_changed_handler_id = None self.pipeline = None self.start_time = 0 self.state = "stopped" self.buffer_count = 0 self.byte_count = 0 self._status_ready = " ScarlettListener is ready" self._status_kw_match = " ScarlettListener caught a keyword match" self._status_cmd_match = " ScarlettListener caught a command match" self._status_stt_failed = " ScarlettListener hit Max STT failures" self._status_cmd_start = " ScarlettListener emitting start command" self._status_cmd_fin = " ScarlettListener Emitting Command run finish" self._status_cmd_cancel = " ScarlettListener cancel speech Recognition" if self.debug: # NOTE: For testing puposes, mainly when in public # so you dont have to keep yelling scarlett in front of strangers self.kw_to_find = ["yo", "hello", "man", "children"] else: # NOTE: Before we start worrying about the config class, lets hardcode what we care about # ADD ME BACK IN WHEN WE REALLY START UNIT TESTING # self.kw_to_find = self.config.get('scarlett', 'keywords') self.kw_to_find = ["scarlett", "SCARLETT"] if self.read_exc: # An error occurred before the stream became ready. self.close(True) raise self.read_exc # pylint: disable=raising-bad-type def scarlett_reset_listen(self): self.failed = 0 self.kw_found = 0 def on_cancel_listening(self, *args, **kwargs): logger.debug("Inside cancel_listening function") self.scarlett_reset_listen() logger.debug("self.failed = {}".format(self.failed)) logger.debug("self.keyword_identified = {}".format(self.kw_found)) def play(self): p = self.pipelines_stack[0] self.state = "active" self.running = True # GST_STATE_PAUSED is the state in which an element is ready to accept and handle data. # For most elements this state is the same as PLAYING. The only exception to this rule are sink elements. # Sink elements only accept one single buffer of data and then block. # At this point the pipeline is 'prerolled' and ready to render data immediately. p.set_state(Gst.State.PAUSED) # GST_STATE_PLAYING is the highest state that an element can be in. # For most elements this state is exactly the same as PAUSED, # they accept and process events and buffers with data. # Only sink elements need to differentiate between PAUSED and PLAYING state. # In PLAYING state, sink elements actually render incoming data, # e.g. output audio to a sound card or render video pictures to an image sink. ret = p.set_state(Gst.State.PLAYING) if ret == Gst.StateChangeReturn.FAILURE: logger.error( "ERROR: Unable to set the pipeline to the playing state") # 8/8/2018 (Only enable this if we turn on debug mode) if os.environ.get("SCARLETT_DEBUG_MODE"): self.on_debug_activate() logger.debug("BEFORE: self.ready_sem.acquire()") self.ready_sem.acquire() logger.debug("AFTER: self.ready_sem.acquire()") logger.info("Press Ctrl+C to quit ...") def stop(self): p = self.pipelines_stack[0] self.state = "stopped" self.running = False # GST_STATE_NULL is the default state of an element. # In this state, it has not allocated any runtime resources, # it has not loaded any runtime libraries and it can obviously not handle data. p.set_state(Gst.State.NULL) def get_pocketsphinx_definition(self, override=False): r""" GST_DEBUG=2,pocketsphinx*:5 gst-launch-1.0 alsasrc device=plughw:CARD=Device,DEV=0 ! \ queue name=capsfilter_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ capsfilter caps='audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved' ! \ audioconvert ! \ audioresample ! \ pocketsphinx \ name=asr \ lm=~/dev/bossjones-github/scarlett_os/static/speech/lm/1473.lm \ dict=~/dev/bossjones-github/scarlett_os/static/speech/dict/1473.dic \ hmm=~/.virtualenvs/scarlett_os/share/pocketsphinx/model/en-us/en-us bestpath=true ! \ tee name=tee ! \ queue name=appsink_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ appsink caps='audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved' \ drop=false max-buffers=10 sync=false \ emit-signals=true tee. queue name=fakesink_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ fakesink sync=false """ logger.debug("Inside get_pocketsphinx_definition") if override: _gst_launch = override else: # TODO: Add audio levels, see the following # SOURCE: http://stackoverflow.com/questions/5686424/detecting-blowing-on-a-microphone-with-gstreamer-or-another-library _gst_launch = [ "alsasrc device=" + self.device, # source: https://github.com/walterbender/story/blob/master/grecord.py # without a buffer here, gstreamer struggles at the start of the # recording and then the A/V sync is bad for the whole video # (possibly a gstreamer/ALSA bug -- even if it gets caught up, it # should be able to resync without problem) # 'progressreport name=progressreport update-freq=1', # NOTE: comment this in when you want performance information "queue name=capsfilter_queue silent=false leaky=2 max-size-buffers=0 max-size-time=0 max-size-bytes=0", "capsfilter name=capsfilter caps=audio/x-raw,format=S16LE,channels=1,layout=interleaved", "audioconvert name=audioconvert", "audioresample name=audioresample", "identity name=ident", "pocketsphinx name=asr", "tee name=tee", "queue name=appsink_queue silent=false leaky=2 max-size-buffers=0 max-size-time=0 max-size-bytes=0", # caps=audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved # NOQA "appsink name=appsink drop=false max-buffers=10 sync=false emit-signals=true tee.", "queue leaky=2 name=fakesink_queue", "fakesink", ] return _gst_launch def cancel(self): """ Threads in python are not cancellable, so we implement our own cancellation logic """ self.cancelled = True @abort_on_exception def run(self, event=None): # TODO: WE NEED TO USE A THREADING EVENT OR A RLOCK HERE TO WAIT TILL DBUS SERVICE IS RUNNING TO CONNECT # TODO: SIGNALS TO THE DBUS PROXY METHODS WE WANT TO USE # TODO: lock.acquire() / event / condition # TODO: self.connect_to_dbus() # TODO: self.setup_dbus_callbacks_handlers() self._connect_to_dbus() self.init_gst() print("Running {}".format(str(self))) self.play() self.emit("playback-status-changed") self.emit("playing-changed") # FIXME: is this needed? # self.mainloop.run() def _connect_to_dbus(self): self.bus = SessionBus() self.dbus_proxy = self.bus.get( "org.scarlett", object_path="/org/scarlett/Listener") # NOQA self.dbus_proxy.emitConnectedToListener("ScarlettListener") time.sleep(2) logger.info("_connect_to_dbus") # TODO: Add a ss_cancel_signal.disconnect() function later ss_cancel_signal = self.bus.subscribe( sender=None, iface="org.scarlett.Listener", signal="ListenerCancelSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=self.on_cancel_listening, ) # NOTE: This function generates the dot file, checks that graphviz in installed and # then finally generates a png file, which it then displays def on_debug_activate(self): # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations # FIXME: STATIC PATH 7/3/2018 # dotfile = ( # "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener.dot" # ) # pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener-pipeline.png" # NOQA dotfile = self._dotfile_listener pngfile = self._pngfile_listener if os.access(dotfile, os.F_OK): os.remove(dotfile) if os.access(pngfile, os.F_OK): os.remove(pngfile) Gst.debug_bin_to_dot_file(self.pipelines_stack[0], Gst.DebugGraphDetails.ALL, "generator-listener") cmd = "/usr/bin/dot -Tpng -o {pngfile} {dotfile}".format( pngfile=pngfile, dotfile=dotfile) os.system(cmd) def result(self, final_hyp): """Forward result signals on the bus to the main thread.""" logger.debug("Inside result function") logger.debug("final_hyp: {}".format(final_hyp)) pp.pprint(final_hyp) logger.debug("kw_to_find: {}".format(self.kw_to_find)) if final_hyp in self.kw_to_find and final_hyp != "": logger.debug("HYP-IS-SOMETHING: {}\n\n\n".format(final_hyp)) self.failed = 0 self.kw_found = 1 self.dbus_proxy.emitKeywordRecognizedSignal() # CHANGEME else: failed_temp = self.failed + 1 self.failed = failed_temp logger.debug("self.failed = {}".format(int(self.failed))) # failed > 10 if self.failed > 4: # reset pipline self.dbus_proxy.emitSttFailedSignal() # CHANGEME self.scarlett_reset_listen() def run_cmd(self, final_hyp): logger.debug("Inside run_cmd function") logger.debug("KEYWORD IDENTIFIED BABY") logger.debug("self.kw_found = {}".format(int(self.kw_found))) if final_hyp == "CANCEL": self.dbus_proxy.emitListenerCancelSignal() # CHANGEME self.on_cancel_listening() else: current_kw_identified = self.kw_found self.kw_found = current_kw_identified self.dbus_proxy.emitCommandRecognizedSignal(final_hyp) # CHANGEME logger.info("Command = {}".format(final_hyp)) logger.debug("AFTER run_cmd, self.kw_found = {}".format( int(self.kw_found))) def init_gst(self): logger.debug("Inside init_gst") self.start_time = time.time() pipeline = Gst.parse_launch(" ! ".join( self.get_pocketsphinx_definition())) logger.debug("After get_pocketsphinx_definition") # Add pipeline obj to stack we can pull from later self.pipelines_stack.append(pipeline) gst_bus = pipeline.get_bus() # gst_bus = pipeline.get_gst_bus() gst_bus.add_signal_watch() self.bus_message_element_handler_id = gst_bus.connect( "message::element", self._on_message) self.bus_message_eos_handler_id = gst_bus.connect( "message::eos", self._on_message) self.bus_message_error_handler_id = gst_bus.connect( "message::error", self._on_message) self.bus_message_state_changed_handler_id = gst_bus.connect( "message::state-changed", self._on_state_changed) # Add bus obj to stack we can pull from later self.gst_bus_stack.append(gst_bus) appsink = pipeline.get_by_name("appsink") appsink.set_property( "caps", Gst.Caps.from_string( "audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved" ), ) appsink.set_property("drop", False) appsink.set_property("max-buffers", BUFFER_SIZE) appsink.set_property("sync", False) # The callback to receive decoded data. appsink.set_property("emit-signals", True) appsink.connect("new-sample", self._new_sample) self.caps_handler = appsink.get_static_pad("sink").connect( "notify::caps", self._notify_caps) self.elements_stack.append(appsink) # ************************************************************ # get gst pipeline element pocketsphinx and set properties - BEGIN # ************************************************************ pocketsphinx = pipeline.get_by_name("asr") # from scarlett_os.internal.debugger import dump # print("debug-2018-pocketsphinx - BEGIN") # dump(pocketsphinx.get_property('decoder')) # print("debug-2018-pocketsphinx - END") # print(pocketsphinx.list_properties()) if self._hmm: pocketsphinx.set_property("hmm", self._hmm) if self._lm: pocketsphinx.set_property("lm", self._lm) if self._dic: pocketsphinx.set_property("dict", self._dic) if self._fwdflat: pocketsphinx.set_property("fwdflat", self._fwdflat) if self._bestpath: pocketsphinx.set_property("bestpath", self._bestpath) if self._dsratio: pocketsphinx.set_property("dsratio", self._dsratio) if self._maxhmmpf: pocketsphinx.set_property("maxhmmpf", self._maxhmmpf) if self._bestpath: pocketsphinx.set_property("bestpath", self._bestpath) # if self._silprob: # pocketsphinx.set_property("silprob", self._silprob) # if self._wip: # pocketsphinx.set_property("wip", self._wip) # ************************************************************ # get gst pipeline element pocketsphinx and set properties - END # ************************************************************ # NOTE: Old way of setting pocketsphinx properties. 8/5/2018 # pocketsphinx.set_property( # "fwdflat", True # ) # Enable Flat Lexicon Search | Default: true # pocketsphinx.set_property( # "bestpath", True # ) # Enable Graph Search | Boolean. Default: true # pocketsphinx.set_property( # "dsratio", 1 # ) # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 # pocketsphinx.set_property( # "maxhmmpf", 3000 # ) # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 # pocketsphinx.set_property( # "bestpath", True # ) # Enable Graph Search | Boolean. Default: true # pocketsphinx.set_property('maxwpf', -1) # # pocketsphinx.set_property('maxwpf', 20) # Maximum number of words # searched per frame | Range: 1 - 100000 Default: -1 self.elements_stack.append(pocketsphinx) capsfilter_queue = pipeline.get_by_name("capsfilter_queue") capsfilter_queue.set_property("leaky", True) # prefer fresh data capsfilter_queue.set_property("silent", False) capsfilter_queue.set_property("max-size-time", 0) # 0 seconds capsfilter_queue.set_property("max-size-buffers", 0) capsfilter_queue.set_property("max-size-bytes", 0) self.capsfilter_queue_overrun_handler_id = capsfilter_queue.connect( "overrun", self._log_queue_overrun) # capsfilter_queue.connect('overrun', self._on_overrun) # capsfilter_queue.connect('underrun', self._on_underrun) # capsfilter_queue.connect('pushing', self._on_pushing) # capsfilter_queue.connect('running', self._on_running) self.elements_stack.append(capsfilter_queue) ident = pipeline.get_by_name("ident") # ident.connect('handoff', self._on_handoff) self.elements_stack.append(ident) logger.debug("After all self.elements_stack.append() calls") # Set up the queue for data and run the main thread. self.queue = queue.Queue(QUEUE_SIZE) self.thread = get_loop_thread() # NOTE: Disabled since we aren't connecting to handoff # def _on_handoff(self, element, buf): # logger.debug('buf:') # pp.pprint(buf) # pp.pprint(dir(buf)) # logger.debug("on_handoff - %d bytes".format(len(buf)) # if self.signed is None: # # only ever one caps struct on our buffers # struct = buf.get_caps().get_structure(0) # # I think these are always set too, but catch just in case # try: # self.signed = struct["signed"] # self.depth = struct["depth"] # self.rate = struct["rate"] # self.channels = struct["channels"] # except Exception: # logger.debug('on_handoff: missing caps') # pass # raw = str(buf) # # # print 'len(raw) =', len(raw) # # sm = 0 # for i in range(0, len(raw)): # sm += ord(raw[i]) # # print sm # FIXEME: Add somthing like analyse.py # SOURCE: https://github.com/jcupitt/huebert/blob/master/huebert/audio.py def _on_state_changed(self, bus, msg): states = msg.parse_state_changed() # To state is PLAYING if msg.src.get_name() == "pipeline0" and states[1] == 4: logger.info("Inside pipeline0 on _on_state_changed") logger.info("State: {}".format(states[1])) self.ready_sem.release() return False else: # logger.error('NOTHING RETURNED in _on_state_changed') logger.info("State: {}".format(states[1])) def _on_overrun(self, element): logging.debug("on_overrun") def _on_underrun(self, element): logging.debug("on_underrun") def _on_running(self, element): logging.debug("on_running") def _on_pushing(self, element): logging.debug("on_pushing") def _notify_caps(self, pad, args): """The callback for the sinkpad's "notify::caps" signal. """ # The sink has started to receive data, so the stream is ready. # This also is our opportunity to read information about the # stream. self.got_caps = True # Allow constructor to complete. self.ready_sem.release() _got_a_pad = False def _log_queue_overrun(self, queue): cbuffers = queue.get_property("current-level-buffers") cbytes = queue.get_property("current-level-bytes") ctime = queue.get_property("current-level-time") def _new_sample(self, sink): """The callback for appsink's "new-sample" signal. """ if self.running: # New data is available from the pipeline! Dump it into our # queue (or possibly block if we're full). buf = sink.emit("pull-sample").get_buffer() # IMPORTANT!!!!! # NOTE: I think this is causing a deadlock self.queue.put(buf.extract_dup(0, buf.get_size())) # "OK = 0. Data passing was ok."" return Gst.FlowReturn.OK def _on_message(self, bus, message): """The callback for GstBus's "message" signal (for two kinds of messages). """ # logger.debug("[_on_message](%s, %s)", bus, message) if not self.finished: struct = message.get_structure() if message.type == Gst.MessageType.EOS: # The file is done. Tell the consumer thread. self.queue.put(SENTINEL) if not self.got_caps: logger.error( "If the stream ends before _notify_caps was called, this is an invalid stream." ) # If the stream ends before _notify_caps was called, this # is an invalid file. self.read_exc = NoStreamError() self.ready_sem.release() elif struct and struct.get_name() == "pocketsphinx": # "final", G_TYPE_BOOLEAN, final, # SOURCE: https://github.com/cmusphinx/pocketsphinx/blob/1fdc9ccb637836d45d40956e745477a2bd3b470a/src/gst-plugin/gstpocketsphinx.c if struct["final"]: logger.info("confidence: {}".format(struct["confidence"])) logger.info("hypothesis: {}".format(struct["hypothesis"])) if self.kw_found == 1: # If keyword is set AND qualifier # then perform action self.run_cmd(struct["hypothesis"]) else: # If it's the main keyword, # set values wait for qualifier self.result(struct["hypothesis"]) elif message.type == Gst.MessageType.ERROR: gerror, debug = message.parse_error() pp.pprint(("gerror,debug:", gerror, debug)) if "not-linked" in debug: logger.error("not-linked") self.read_exc = NoStreamError() elif "No such device" in debug: logger.error("No such device") self.read_exc = NoStreamError() else: logger.info("FileReadError") pp.pprint(("SOME FileReadError", bus, message, struct, struct.get_name())) self.read_exc = FileReadError(debug) self.ready_sem.release() # Cleanup. def close(self, force=False): """Close the file and clean up associated resources. Calling `close()` a second time has no effect. """ if self.running or force: self.running = False self.finished = True try: gst_bus = self.gst_bus_stack[0] except Exception: logger.error("Failed to get gst_bus from gst_bus_stack[0]") if gst_bus: gst_bus.remove_signal_watch() if self.bus_message_element_handler_id: gst_bus.disconnect(self.bus_message_element_handler_id) if self.bus_message_error_handler_id: gst_bus.disconnect(self.bus_message_error_handler_id) if self.bus_message_eos_handler_id: gst_bus.disconnect(self.bus_message_eos_handler_id) if self.bus_message_state_changed_handler_id: gst_bus.disconnect( self.bus_message_state_changed_handler_id) self.bus = None self.pipeline = None self.codec = None self.bitrate = -1 self.state = None # Unregister for signals, which we registered for above with # `add_signal_watch`. (Without this, GStreamer leaks file # descriptors.) logger.debug("BEFORE p = self.pipelines_stack[0]") p = self.pipelines_stack[0] p.get_bus().remove_signal_watch() logger.debug("AFTER p.get_bus().remove_signal_watch()") # Block spurious signals. appsink = self.elements_stack[0] appsink.get_static_pad("sink").disconnect(self.caps_handler) # Make space in the output queue to let the decoder thread # finish. (Otherwise, the thread blocks on its enqueue and # the interpreter hangs.) try: self.queue.get_nowait() except queue.Empty: pass # Halt the pipeline (closing file). self.stop() # Delete the pipeline object. This seems to be necessary on Python # 2, but not Python 3 for some reason: on 3.5, at least, the # pipeline gets dereferenced automatically. del p def __del__(self): self.close() def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False
class ScarlettListenerI(threading.Thread, _IdleObject): """ Attempt to take out all Gstreamer logic and put it in a class ouside the dbus server. Cancellable thread which uses gobject signals to return information to the GUI. """ __gsignals__ = SCARLETT_LISTENER_I_SIGNALS device = PS_DEVICE hmm = HMM_PATH lm = LM_PATH dic = DICT_PATH # __dr = None __instance = None MAX_FAILURES = 10 def __init__(self, name, config_manager, *args): threading.Thread.__init__(self) _IdleObject.__init__(self) self.running = False self.finished = False self.ready_sem = threading.Semaphore(SEMAPHORE_NUM) self.queue = queue.Queue(QUEUE_SIZE) # Load in config object, and set default device information self._config_manager = config_manager self._graphviz_debug_dir = self._config_manager.cfg["graphviz_debug_dir"] self._device = self._config_manager.cfg["pocketsphinx"]["device"] self._hmm = self._config_manager.cfg["pocketsphinx"]["hmm"] self._lm = self._config_manager.cfg["pocketsphinx"]["lm"] self._dic = self._config_manager.cfg["pocketsphinx"]["dict"] self._fwdflat = bool(self._config_manager.cfg["pocketsphinx"]["fwdflat"]) self._bestpath = bool(self._config_manager.cfg["pocketsphinx"]["bestpath"]) self._dsratio = int(self._config_manager.cfg["pocketsphinx"]["dsratio"]) self._maxhmmpf = int(self._config_manager.cfg["pocketsphinx"]["maxhmmpf"]) self._bestpath = bool(self._config_manager.cfg["pocketsphinx"]["bestpath"]) self._silprob = float(self._config_manager.cfg["pocketsphinx"]["silprob"]) self._wip = float(self._config_manager.cfg["pocketsphinx"]["wip"]) # dotfile setup self._dotfile_listener = os.path.join( self._graphviz_debug_dir, "generator-listener.dot" ) self._pngfile_listener = os.path.join( self._graphviz_debug_dir, "generator-listener-pipeline.png" ) # self._handler = DbusSignalHandler() # Get a dbus proxy and check if theres a service registered called 'org.scarlett.Listener' # if not, then we can skip all further processing. (The scarlett-os-mpris-dbus seems not to be running) # self.__dr = DBusRunner.get_instance() logger.info("Initializing ScarlettListenerI") # This wil get filled with an exception if opening fails. self.read_exc = None self.dot_exc = None self.cancelled = False self.name = name self.setName("{}".format(self.name)) self.pipelines_stack = [] self.elements_stack = [] self.gst_bus_stack = [] self._message = "This is the ScarlettListenerI" # TODO: When we're ready to unit test, config this back in!!!!! # self.config = scarlett_config.Config() self.config = None self.override_parse = "" self.failed = 0 self.kw_found = 0 self.debug = False self.create_dot = True self.terminate = False self.capsfilter_queue_overrun_handler_id = None self._cancel_signal_callback = None # source: https://github.com/ljmljz/xpra/blob/b32f748e0c29cdbfab836b3901c1e318ea142b33/src/xpra/sound/sound_pipeline.py # NOQA self.bus = None self.bus_message_element_handler_id = None self.bus_message_error_handler_id = None self.bus_message_eos_handler_id = None self.bus_message_state_changed_handler_id = None self.pipeline = None self.start_time = 0 self.state = "stopped" self.buffer_count = 0 self.byte_count = 0 self._status_ready = " ScarlettListener is ready" self._status_kw_match = " ScarlettListener caught a keyword match" self._status_cmd_match = " ScarlettListener caught a command match" self._status_stt_failed = " ScarlettListener hit Max STT failures" self._status_cmd_start = " ScarlettListener emitting start command" self._status_cmd_fin = " ScarlettListener Emitting Command run finish" self._status_cmd_cancel = " ScarlettListener cancel speech Recognition" if self.debug: # NOTE: For testing puposes, mainly when in public # so you dont have to keep yelling scarlett in front of strangers self.kw_to_find = ["yo", "hello", "man", "children"] else: # NOTE: Before we start worrying about the config class, lets hardcode what we care about # ADD ME BACK IN WHEN WE REALLY START UNIT TESTING # self.kw_to_find = self.config.get('scarlett', 'keywords') self.kw_to_find = ["scarlett", "SCARLETT"] if self.read_exc: # An error occurred before the stream became ready. self.close(True) raise self.read_exc # pylint: disable=raising-bad-type def scarlett_reset_listen(self): self.failed = 0 self.kw_found = 0 def on_cancel_listening(self, *args, **kwargs): logger.debug("Inside cancel_listening function") self.scarlett_reset_listen() logger.debug("self.failed = {}".format(self.failed)) logger.debug("self.keyword_identified = {}".format(self.kw_found)) def play(self): p = self.pipelines_stack[0] self.state = "active" self.running = True # GST_STATE_PAUSED is the state in which an element is ready to accept and handle data. # For most elements this state is the same as PLAYING. The only exception to this rule are sink elements. # Sink elements only accept one single buffer of data and then block. # At this point the pipeline is 'prerolled' and ready to render data immediately. p.set_state(Gst.State.PAUSED) # GST_STATE_PLAYING is the highest state that an element can be in. # For most elements this state is exactly the same as PAUSED, # they accept and process events and buffers with data. # Only sink elements need to differentiate between PAUSED and PLAYING state. # In PLAYING state, sink elements actually render incoming data, # e.g. output audio to a sound card or render video pictures to an image sink. ret = p.set_state(Gst.State.PLAYING) if ret == Gst.StateChangeReturn.FAILURE: logger.error("ERROR: Unable to set the pipeline to the playing state") # 8/8/2018 (Only enable this if we turn on debug mode) if os.environ.get("SCARLETT_DEBUG_MODE"): self.on_debug_activate() logger.debug("BEFORE: self.ready_sem.acquire()") self.ready_sem.acquire() logger.debug("AFTER: self.ready_sem.acquire()") logger.info("Press Ctrl+C to quit ...") def stop(self): p = self.pipelines_stack[0] self.state = "stopped" self.running = False # GST_STATE_NULL is the default state of an element. # In this state, it has not allocated any runtime resources, # it has not loaded any runtime libraries and it can obviously not handle data. p.set_state(Gst.State.NULL) def get_pocketsphinx_definition(self, override=False): r""" GST_DEBUG=2,pocketsphinx*:5 gst-launch-1.0 alsasrc device=plughw:CARD=Device,DEV=0 ! \ queue name=capsfilter_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ capsfilter caps='audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved' ! \ audioconvert ! \ audioresample ! \ pocketsphinx \ name=asr \ lm=~/dev/bossjones-github/scarlett_os/static/speech/lm/1473.lm \ dict=~/dev/bossjones-github/scarlett_os/static/speech/dict/1473.dic \ hmm=~/.virtualenvs/scarlett_os/share/pocketsphinx/model/en-us/en-us bestpath=true ! \ tee name=tee ! \ queue name=appsink_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ appsink caps='audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved' \ drop=false max-buffers=10 sync=false \ emit-signals=true tee. queue name=fakesink_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ fakesink sync=false """ logger.debug("Inside get_pocketsphinx_definition") if override: _gst_launch = override else: # TODO: Add audio levels, see the following # SOURCE: http://stackoverflow.com/questions/5686424/detecting-blowing-on-a-microphone-with-gstreamer-or-another-library _gst_launch = [ "alsasrc device=" + self.device, # source: https://github.com/walterbender/story/blob/master/grecord.py # without a buffer here, gstreamer struggles at the start of the # recording and then the A/V sync is bad for the whole video # (possibly a gstreamer/ALSA bug -- even if it gets caught up, it # should be able to resync without problem) # 'progressreport name=progressreport update-freq=1', # NOTE: comment this in when you want performance information "queue name=capsfilter_queue silent=false leaky=2 max-size-buffers=0 max-size-time=0 max-size-bytes=0", "capsfilter name=capsfilter caps=audio/x-raw,format=S16LE,channels=1,layout=interleaved", "audioconvert name=audioconvert", "audioresample name=audioresample", "identity name=ident", "pocketsphinx name=asr", "tee name=tee", "queue name=appsink_queue silent=false leaky=2 max-size-buffers=0 max-size-time=0 max-size-bytes=0", # caps=audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved # NOQA "appsink name=appsink drop=false max-buffers=10 sync=false emit-signals=true tee.", "queue leaky=2 name=fakesink_queue", "fakesink", ] return _gst_launch def cancel(self): """ Threads in python are not cancellable, so we implement our own cancellation logic """ self.cancelled = True @abort_on_exception def run(self, event=None): # TODO: WE NEED TO USE A THREADING EVENT OR A RLOCK HERE TO WAIT TILL DBUS SERVICE IS RUNNING TO CONNECT # TODO: SIGNALS TO THE DBUS PROXY METHODS WE WANT TO USE # TODO: lock.acquire() / event / condition # TODO: self.connect_to_dbus() # TODO: self.setup_dbus_callbacks_handlers() self._connect_to_dbus() self.init_gst() print("Running {}".format(str(self))) self.play() self.emit("playback-status-changed") self.emit("playing-changed") # FIXME: is this needed? # self.mainloop.run() def _connect_to_dbus(self): self.bus = SessionBus() self.dbus_proxy = self.bus.get( "org.scarlett", object_path="/org/scarlett/Listener" ) # NOQA self.dbus_proxy.emitConnectedToListener("ScarlettListener") time.sleep(2) logger.info("_connect_to_dbus") # TODO: Add a ss_cancel_signal.disconnect() function later ss_cancel_signal = self.bus.subscribe( sender=None, iface="org.scarlett.Listener", signal="ListenerCancelSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=self.on_cancel_listening, ) # NOTE: This function generates the dot file, checks that graphviz in installed and # then finally generates a png file, which it then displays def on_debug_activate(self): # FIXME: This needs to use dynamic paths, it's possible that we're having issues because of order of operations # FIXME: STATIC PATH 7/3/2018 # dotfile = ( # "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener.dot" # ) # pngfile = "/home/pi/dev/bossjones-github/scarlett_os/_debug/generator-listener-pipeline.png" # NOQA dotfile = self._dotfile_listener pngfile = self._pngfile_listener if os.access(dotfile, os.F_OK): os.remove(dotfile) if os.access(pngfile, os.F_OK): os.remove(pngfile) Gst.debug_bin_to_dot_file( self.pipelines_stack[0], Gst.DebugGraphDetails.ALL, "generator-listener" ) cmd = "/usr/bin/dot -Tpng -o {pngfile} {dotfile}".format( pngfile=pngfile, dotfile=dotfile ) os.system(cmd) def result(self, final_hyp): """Forward result signals on the bus to the main thread.""" logger.debug("Inside result function") logger.debug("final_hyp: {}".format(final_hyp)) pp.pprint(final_hyp) logger.debug("kw_to_find: {}".format(self.kw_to_find)) if final_hyp in self.kw_to_find and final_hyp != "": logger.debug("HYP-IS-SOMETHING: {}\n\n\n".format(final_hyp)) self.failed = 0 self.kw_found = 1 self.dbus_proxy.emitKeywordRecognizedSignal() # CHANGEME else: failed_temp = self.failed + 1 self.failed = failed_temp logger.debug("self.failed = {}".format(int(self.failed))) # failed > 10 if self.failed > 4: # reset pipline self.dbus_proxy.emitSttFailedSignal() # CHANGEME self.scarlett_reset_listen() def run_cmd(self, final_hyp): logger.debug("Inside run_cmd function") logger.debug("KEYWORD IDENTIFIED BABY") logger.debug("self.kw_found = {}".format(int(self.kw_found))) if final_hyp == "CANCEL": self.dbus_proxy.emitListenerCancelSignal() # CHANGEME self.on_cancel_listening() else: current_kw_identified = self.kw_found self.kw_found = current_kw_identified self.dbus_proxy.emitCommandRecognizedSignal(final_hyp) # CHANGEME logger.info("Command = {}".format(final_hyp)) logger.debug("AFTER run_cmd, self.kw_found = {}".format(int(self.kw_found))) def init_gst(self): logger.debug("Inside init_gst") self.start_time = time.time() pipeline = Gst.parse_launch(" ! ".join(self.get_pocketsphinx_definition())) logger.debug("After get_pocketsphinx_definition") # Add pipeline obj to stack we can pull from later self.pipelines_stack.append(pipeline) gst_bus = pipeline.get_bus() # gst_bus = pipeline.get_gst_bus() gst_bus.add_signal_watch() self.bus_message_element_handler_id = gst_bus.connect( "message::element", self._on_message ) self.bus_message_eos_handler_id = gst_bus.connect( "message::eos", self._on_message ) self.bus_message_error_handler_id = gst_bus.connect( "message::error", self._on_message ) self.bus_message_state_changed_handler_id = gst_bus.connect( "message::state-changed", self._on_state_changed ) # Add bus obj to stack we can pull from later self.gst_bus_stack.append(gst_bus) appsink = pipeline.get_by_name("appsink") appsink.set_property( "caps", Gst.Caps.from_string( "audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved" ), ) appsink.set_property("drop", False) appsink.set_property("max-buffers", BUFFER_SIZE) appsink.set_property("sync", False) # The callback to receive decoded data. appsink.set_property("emit-signals", True) appsink.connect("new-sample", self._new_sample) self.caps_handler = appsink.get_static_pad("sink").connect( "notify::caps", self._notify_caps ) self.elements_stack.append(appsink) # ************************************************************ # get gst pipeline element pocketsphinx and set properties - BEGIN # ************************************************************ pocketsphinx = pipeline.get_by_name("asr") # from scarlett_os.internal.debugger import dump # print("debug-2018-pocketsphinx - BEGIN") # dump(pocketsphinx.get_property('decoder')) # print("debug-2018-pocketsphinx - END") # print(pocketsphinx.list_properties()) if self._hmm: pocketsphinx.set_property("hmm", self._hmm) if self._lm: pocketsphinx.set_property("lm", self._lm) if self._dic: pocketsphinx.set_property("dict", self._dic) if self._fwdflat: pocketsphinx.set_property("fwdflat", self._fwdflat) if self._bestpath: pocketsphinx.set_property("bestpath", self._bestpath) if self._dsratio: pocketsphinx.set_property("dsratio", self._dsratio) if self._maxhmmpf: pocketsphinx.set_property("maxhmmpf", self._maxhmmpf) if self._bestpath: pocketsphinx.set_property("bestpath", self._bestpath) # if self._silprob: # pocketsphinx.set_property("silprob", self._silprob) # if self._wip: # pocketsphinx.set_property("wip", self._wip) # ************************************************************ # get gst pipeline element pocketsphinx and set properties - END # ************************************************************ # NOTE: Old way of setting pocketsphinx properties. 8/5/2018 # pocketsphinx.set_property( # "fwdflat", True # ) # Enable Flat Lexicon Search | Default: true # pocketsphinx.set_property( # "bestpath", True # ) # Enable Graph Search | Boolean. Default: true # pocketsphinx.set_property( # "dsratio", 1 # ) # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 # pocketsphinx.set_property( # "maxhmmpf", 3000 # ) # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 # pocketsphinx.set_property( # "bestpath", True # ) # Enable Graph Search | Boolean. Default: true # pocketsphinx.set_property('maxwpf', -1) # # pocketsphinx.set_property('maxwpf', 20) # Maximum number of words # searched per frame | Range: 1 - 100000 Default: -1 self.elements_stack.append(pocketsphinx) capsfilter_queue = pipeline.get_by_name("capsfilter_queue") capsfilter_queue.set_property("leaky", True) # prefer fresh data capsfilter_queue.set_property("silent", False) capsfilter_queue.set_property("max-size-time", 0) # 0 seconds capsfilter_queue.set_property("max-size-buffers", 0) capsfilter_queue.set_property("max-size-bytes", 0) self.capsfilter_queue_overrun_handler_id = capsfilter_queue.connect( "overrun", self._log_queue_overrun ) # capsfilter_queue.connect('overrun', self._on_overrun) # capsfilter_queue.connect('underrun', self._on_underrun) # capsfilter_queue.connect('pushing', self._on_pushing) # capsfilter_queue.connect('running', self._on_running) self.elements_stack.append(capsfilter_queue) ident = pipeline.get_by_name("ident") # ident.connect('handoff', self._on_handoff) self.elements_stack.append(ident) logger.debug("After all self.elements_stack.append() calls") # Set up the queue for data and run the main thread. self.queue = queue.Queue(QUEUE_SIZE) self.thread = get_loop_thread() # NOTE: Disabled since we aren't connecting to handoff # def _on_handoff(self, element, buf): # logger.debug('buf:') # pp.pprint(buf) # pp.pprint(dir(buf)) # logger.debug("on_handoff - %d bytes".format(len(buf)) # if self.signed is None: # # only ever one caps struct on our buffers # struct = buf.get_caps().get_structure(0) # # I think these are always set too, but catch just in case # try: # self.signed = struct["signed"] # self.depth = struct["depth"] # self.rate = struct["rate"] # self.channels = struct["channels"] # except Exception: # logger.debug('on_handoff: missing caps') # pass # raw = str(buf) # # # print 'len(raw) =', len(raw) # # sm = 0 # for i in range(0, len(raw)): # sm += ord(raw[i]) # # print sm # FIXEME: Add somthing like analyse.py # SOURCE: https://github.com/jcupitt/huebert/blob/master/huebert/audio.py def _on_state_changed(self, bus, msg): states = msg.parse_state_changed() # To state is PLAYING if msg.src.get_name() == "pipeline0" and states[1] == 4: logger.info("Inside pipeline0 on _on_state_changed") logger.info("State: {}".format(states[1])) self.ready_sem.release() return False else: # logger.error('NOTHING RETURNED in _on_state_changed') logger.info("State: {}".format(states[1])) def _on_overrun(self, element): logging.debug("on_overrun") def _on_underrun(self, element): logging.debug("on_underrun") def _on_running(self, element): logging.debug("on_running") def _on_pushing(self, element): logging.debug("on_pushing") def _notify_caps(self, pad, args): """The callback for the sinkpad's "notify::caps" signal. """ # The sink has started to receive data, so the stream is ready. # This also is our opportunity to read information about the # stream. self.got_caps = True # Allow constructor to complete. self.ready_sem.release() _got_a_pad = False def _log_queue_overrun(self, queue): cbuffers = queue.get_property("current-level-buffers") cbytes = queue.get_property("current-level-bytes") ctime = queue.get_property("current-level-time") def _new_sample(self, sink): """The callback for appsink's "new-sample" signal. """ if self.running: # New data is available from the pipeline! Dump it into our # queue (or possibly block if we're full). buf = sink.emit("pull-sample").get_buffer() # IMPORTANT!!!!! # NOTE: I think this is causing a deadlock self.queue.put(buf.extract_dup(0, buf.get_size())) # "OK = 0. Data passing was ok."" return Gst.FlowReturn.OK def _on_message(self, bus, message): """The callback for GstBus's "message" signal (for two kinds of messages). """ # logger.debug("[_on_message](%s, %s)", bus, message) if not self.finished: struct = message.get_structure() if message.type == Gst.MessageType.EOS: # The file is done. Tell the consumer thread. self.queue.put(SENTINEL) if not self.got_caps: logger.error( "If the stream ends before _notify_caps was called, this is an invalid stream." ) # If the stream ends before _notify_caps was called, this # is an invalid file. self.read_exc = NoStreamError() self.ready_sem.release() elif struct and struct.get_name() == "pocketsphinx": # "final", G_TYPE_BOOLEAN, final, # SOURCE: https://github.com/cmusphinx/pocketsphinx/blob/1fdc9ccb637836d45d40956e745477a2bd3b470a/src/gst-plugin/gstpocketsphinx.c if struct["final"]: logger.info("confidence: {}".format(struct["confidence"])) logger.info("hypothesis: {}".format(struct["hypothesis"])) if self.kw_found == 1: # If keyword is set AND qualifier # then perform action self.run_cmd(struct["hypothesis"]) else: # If it's the main keyword, # set values wait for qualifier self.result(struct["hypothesis"]) elif message.type == Gst.MessageType.ERROR: gerror, debug = message.parse_error() pp.pprint(("gerror,debug:", gerror, debug)) if "not-linked" in debug: logger.error("not-linked") self.read_exc = NoStreamError() elif "No such device" in debug: logger.error("No such device") self.read_exc = NoStreamError() else: logger.info("FileReadError") pp.pprint( ("SOME FileReadError", bus, message, struct, struct.get_name()) ) self.read_exc = FileReadError(debug) self.ready_sem.release() # Cleanup. def close(self, force=False): """Close the file and clean up associated resources. Calling `close()` a second time has no effect. """ if self.running or force: self.running = False self.finished = True try: gst_bus = self.gst_bus_stack[0] except Exception: logger.error("Failed to get gst_bus from gst_bus_stack[0]") if gst_bus: gst_bus.remove_signal_watch() if self.bus_message_element_handler_id: gst_bus.disconnect(self.bus_message_element_handler_id) if self.bus_message_error_handler_id: gst_bus.disconnect(self.bus_message_error_handler_id) if self.bus_message_eos_handler_id: gst_bus.disconnect(self.bus_message_eos_handler_id) if self.bus_message_state_changed_handler_id: gst_bus.disconnect(self.bus_message_state_changed_handler_id) self.bus = None self.pipeline = None self.codec = None self.bitrate = -1 self.state = None # Unregister for signals, which we registered for above with # `add_signal_watch`. (Without this, GStreamer leaks file # descriptors.) logger.debug("BEFORE p = self.pipelines_stack[0]") p = self.pipelines_stack[0] p.get_bus().remove_signal_watch() logger.debug("AFTER p.get_bus().remove_signal_watch()") # Block spurious signals. appsink = self.elements_stack[0] appsink.get_static_pad("sink").disconnect(self.caps_handler) # Make space in the output queue to let the decoder thread # finish. (Otherwise, the thread blocks on its enqueue and # the interpreter hangs.) try: self.queue.get_nowait() except queue.Empty: pass # Halt the pipeline (closing file). self.stop() # Delete the pipeline object. This seems to be necessary on Python # 2, but not Python 3 for some reason: on 3.5, at least, the # pipeline gets dereferenced automatically. del p def __del__(self): self.close() def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False
class ScarlettListenerI(threading.Thread, _IdleObject): """ Attempt to take out all Gstreamer logic and put it in a class ouside the dbus server. Cancellable thread which uses gobject signals to return information to the GUI. """ __gsignals__ = SCARLETT_LISTENER_I_SIGNALS device = PS_DEVICE hmm = HMM_PATH lm = LM_PATH dic = DICT_PATH # SCARLETT_LISTENER_I_SIGNALS = { # "completed": ( # GObject.SignalFlags.RUN_LAST, None, []), # "progress": ( # GObject.SignalFlags.RUN_LAST, None, [ # GObject.TYPE_FLOAT]), # percent complete # "eos": (GObject.SignalFlags.RUN_LAST, None, ()), # "error": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "died": (GObject.SignalFlags.RUN_LAST, None, ()), # "async-done": (GObject.SignalFlags.RUN_LAST, None, ()), # "state-change": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_INT, GObject.TYPE_INT)), # # FIXME: AUDIT THE RETURN TYPES # "keyword-recgonized": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "command-recgonized": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "stt-failed": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "listener-cancel": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "listener-ready": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "connected-to-server": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # "listener-message": (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_STRING, GObject.TYPE_STRING)), # 'finished': (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_PYOBJECT,)), # 'aborted': (GObject.SignalFlags.RUN_LAST, None, (GObject.TYPE_PYOBJECT,)) # } def __init__(self, *args): threading.Thread.__init__(self) _IdleObject.__init__(self) # Gst.init(None) self.running = False self.finished = False self.ready_sem = threading.Semaphore(0) self.queue = queue.Queue(QUEUE_SIZE) # This wil get filled with an exception if opening fails. self.read_exc = None self.dot_exc = None self.cancelled = False self.name = args[0] self.setName("%s" % self.name) self.pipelines_stack = [] self.elements_stack = [] self.gst_bus_stack = [] self._message = 'This is the ScarlettListenerI' self.config = scarlett_config.Config() self.override_parse = '' self.failed = 0 self.kw_found = 0 self.debug = False self.create_dot = True self.terminate = False self.capsfilter_queue_overrun_handler = None # source: https://github.com/ljmljz/xpra/blob/b32f748e0c29cdbfab836b3901c1e318ea142b33/src/xpra/sound/sound_pipeline.py # NOQA self.bus = None self.bus_message_element_handler_id = None self.bus_message_error_handler_id = None self.bus_message_eos_handler_id = None self.bus_message_state_changed_handler_id = None self.pipeline = None self.start_time = 0 self.state = "stopped" self.buffer_count = 0 self.byte_count = 0 # # Thread manager, maximum of 1 since it'll be long running # self.manager = FooThreadManager(1) self._status_ready = " ScarlettListener is ready" self._status_kw_match = " ScarlettListener caught a keyword match" self._status_cmd_match = " ScarlettListener caught a command match" self._status_stt_failed = " ScarlettListener hit Max STT failures" self._status_cmd_start = " ScarlettListener emitting start command" self._status_cmd_fin = " ScarlettListener Emitting Command run finish" self._status_cmd_cancel = " ScarlettListener cancel speech Recognition" if self.debug: # NOTE: For testing puposes, mainly when in public # so you dont have to keep yelling scarlett in front of strangers self.kw_to_find = ['yo', 'hello', 'man', 'children'] else: self.kw_to_find = self.config.get('scarlett', 'keywords') if self.read_exc: # An error occurred before the stream became ready. self.close(True) raise self.read_exc def connect_to_dbus(self): # self.dbus_stack.append(bus) # self.dbus_stack.append(path) # logger.debug("Inside self.dbus_stack") # pp.pprint(self.dbus_stack) pass def scarlett_reset_listen(self): self.failed = 0 self.kw_found = 0 def cancel_listening(self, *args, **kwargs): logger.debug("Inside cancel_listening function") self.scarlett_reset_listen() logger.debug("self.failed = %i" % (self.failed)) logger.debug( "self.keyword_identified = %i" % (self.kw_found)) def play(self): p = self.pipelines_stack[0] self.state = "active" self.running = True # GST_STATE_PAUSED is the state in which an element is ready to accept and handle data. # For most elements this state is the same as PLAYING. The only exception to this rule are sink elements. # Sink elements only accept one single buffer of data and then block. # At this point the pipeline is 'prerolled' and ready to render data immediately. p.set_state(Gst.State.PAUSED) # GST_STATE_PLAYING is the highest state that an element can be in. # For most elements this state is exactly the same as PAUSED, # they accept and process events and buffers with data. # Only sink elements need to differentiate between PAUSED and PLAYING state. # In PLAYING state, sink elements actually render incoming data, # e.g. output audio to a sound card or render video pictures to an image sink. ret = p.set_state(Gst.State.PLAYING) if ret == Gst.StateChangeReturn.FAILURE: logger.error("ERROR: Unable to set the pipeline to the playing state") self.on_debug_activate() logger.debug("BEFORE: self.ready_sem.acquire()") self.ready_sem.acquire() logger.debug("AFTER: self.ready_sem.acquire()") logger.info("Press Ctrl+C to quit ...") # @trace def stop(self): p = self.pipelines_stack[0] self.state = "stopped" self.running = False # GST_STATE_NULL is the default state of an element. # In this state, it has not allocated any runtime resources, # it has not loaded any runtime libraries and it can obviously not handle data. p.set_state(Gst.State.NULL) def get_pocketsphinx_definition(self, override=False): """ GST_DEBUG=2,pocketsphinx*:5 gst-launch-1.0 alsasrc device=plughw:CARD=Device,DEV=0 ! \ queue name=capsfilter_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ capsfilter caps='audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved' ! \ audioconvert ! \ audioresample ! \ pocketsphinx \ name=asr \ lm=~/dev/bossjones-github/scarlett-dbus-poc/tests/fixtures/lm/1473.lm \ dict=~/dev/bossjones-github/scarlett-dbus-poc/tests/fixtures/dict/1473.dic \ hmm=~/.virtualenvs/scarlett-dbus-poc/share/pocketsphinx/model/en-us/en-us bestpath=true ! \ tee name=tee ! \ queue name=appsink_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ appsink caps='audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved' \ drop=false max-buffers=10 sync=false \ emit-signals=true tee. queue name=fakesink_queue \ leaky=2 \ max-size-buffers=0 \ max-size-time=0 \ max-size-bytes=0 ! \ fakesink sync=false """ logger.debug("Inside get_pocketsphinx_definition") if override: _gst_launch = override else: _gst_launch = ['alsasrc device=' + ScarlettListenerI.device, # source: https://github.com/walterbender/story/blob/master/grecord.py # without a buffer here, gstreamer struggles at the start of the # recording and then the A/V sync is bad for the whole video # (possibly a gstreamer/ALSA bug -- even if it gets caught up, it # should be able to resync without problem) 'queue name=capsfilter_queue silent=false leaky=2 max-size-buffers=0 max-size-time=0 max-size-bytes=0', 'capsfilter name=capsfilter caps=audio/x-raw,format=S16LE,channels=1,layout=interleaved', 'audioconvert name=audioconvert', 'audioresample name=audioresample', 'identity name=ident', 'pocketsphinx name=asr', 'tee name=tee', 'queue name=appsink_queue silent=false leaky=2 max-size-buffers=0 max-size-time=0 max-size-bytes=0', # caps=audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved # NOQA 'appsink name=appsink drop=false max-buffers=10 sync=false emit-signals=true tee.', 'queue leaky=2 name=fakesink_queue', 'fakesink'] return _gst_launch # @trace def cancel(self): """ Threads in python are not cancellable, so we implement our own cancellation logic """ self.cancelled = True @abort_on_exception def run(self, event=None): # TODO: WE NEED TO USE A THREADING EVENT OR A RLOCK HERE TO WAIT TILL DBUS SERVICE IS RUNNING TO CONNECT # TODO: SIGNALS TO THE DBUS PROXY METHODS WE WANT TO USE # TODO: lock.acquire() / event / condition # TODO: self.connect_to_dbus() # TODO: self.setup_dbus_callbacks_handlers() self._connect_to_dbus() self.init_gst() print "Running %s" % str(self) self.play() self.emit('playback-status-changed') self.emit('playing-changed') # FIXME: is this needed? # self.mainloop.run() def _connect_to_dbus(self): self.bus = SessionBus() self.dbus_proxy = self.bus.get("org.scarlett", object_path='/org/scarlett/Listener') # NOQA self.dbus_proxy.emitConnectedToListener('ScarlettListener') sleep(2) logger.info('_connect_to_dbus') ss_cancel_signal = self.bus.subscribe(sender=None, iface="org.scarlett.Listener", signal="ListenerCancelSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=self.cancel_listening) # NOTE: This function generates the dot file, checks that graphviz in installed and # then finally generates a png file, which it then displays def on_debug_activate(self): dotfile = "/home/pi/dev/bossjones-github/scarlett-dbus-poc/_debug/generator-listener.dot" pngfile = "/home/pi/dev/bossjones-github/scarlett-dbus-poc/_debug/generator-listener-pipeline.png" # NOQA if os.access(dotfile, os.F_OK): os.remove(dotfile) if os.access(pngfile, os.F_OK): os.remove(pngfile) Gst.debug_bin_to_dot_file(self.pipelines_stack[0], Gst.DebugGraphDetails.ALL, "generator-listener") os.system('/usr/bin/dot' + " -Tpng -o " + pngfile + " " + dotfile) def result(self, final_hyp): """Forward result signals on the bus to the main thread.""" logger.debug("Inside result function") logger.debug("final_hyp: {}".format(final_hyp)) pp.pprint(final_hyp) logger.debug("kw_to_find: {}".format(self.kw_to_find)) if final_hyp in self.kw_to_find and final_hyp != '': logger.debug( "HYP-IS-SOMETHING: " + final_hyp + "\n\n\n") self.failed = 0 self.kw_found = 1 self.dbus_proxy.emitKeywordRecognizedSignal() # CHANGEME else: failed_temp = self.failed + 1 self.failed = failed_temp logger.debug( "self.failed = %i" % (self.failed)) if self.failed > 4: # reset pipline self.dbus_proxy.emitSttFailedSignal() # CHANGEME self.scarlett_reset_listen() def run_cmd(self, final_hyp): logger.debug("Inside run_cmd function") logger.debug("KEYWORD IDENTIFIED BABY") logger.debug( "self.kw_found = %i" % (self.kw_found)) if final_hyp == 'CANCEL': self.dbus_proxy.emitListenerCancelSignal() # CHANGEME self.cancel_listening() else: current_kw_identified = self.kw_found self.kw_found = current_kw_identified self.dbus_proxy.emitCommandRecognizedSignal(final_hyp) # CHANGEME logger.info( " Command = {}".format(final_hyp)) logger.debug( "AFTER run_cmd, self.kw_found = %i" % (self.kw_found)) def init_gst(self): logger.debug("Inside init_gst") self.start_time = time.time() pipeline = Gst.parse_launch(' ! '.join(self.get_pocketsphinx_definition())) logger.debug("After get_pocketsphinx_definition") # Add pipeline obj to stack we can pull from later self.pipelines_stack.append(pipeline) gst_bus = pipeline.get_bus() # gst_bus = pipeline.get_gst_bus() gst_bus.add_signal_watch() self.bus_message_element_handler_id = gst_bus.connect("message::element", self._on_message) self.bus_message_eos_handler_id = gst_bus.connect("message::eos", self._on_message) self.bus_message_error_handler_id = gst_bus.connect("message::error", self._on_message) self.bus_message_state_changed_handler_id = gst_bus.connect("message::state-changed", self._on_state_changed) # Add bus obj to stack we can pull from later self.gst_bus_stack.append(gst_bus) appsink = pipeline.get_by_name('appsink') appsink.set_property( 'caps', Gst.Caps.from_string('audio/x-raw,format=(string)S16LE,rate=(int)16000,channels=(int)1,layout=(string)interleaved'), ) appsink.set_property('drop', False) appsink.set_property('max-buffers', BUFFER_SIZE) appsink.set_property('sync', False) # The callback to receive decoded data. appsink.set_property('emit-signals', True) appsink.connect("new-sample", self._new_sample) self.caps_handler = appsink.get_static_pad("sink").connect( "notify::caps", self._notify_caps ) self.elements_stack.append(appsink) # get gst pipeline element pocketsphinx and set properties pocketsphinx = pipeline.get_by_name('asr') if ScarlettListenerI.hmm: pocketsphinx.set_property('hmm', ScarlettListenerI.hmm) if ScarlettListenerI.lm: pocketsphinx.set_property('lm', ScarlettListenerI.lm) if ScarlettListenerI.dic: pocketsphinx.set_property('dict', ScarlettListenerI.dic) pocketsphinx.set_property('fwdflat', True) # Enable Flat Lexicon Search | Default: true pocketsphinx.set_property('bestpath', True) # Enable Graph Search | Boolean. Default: true pocketsphinx.set_property('dsratio', 1) # Evaluate acoustic model every N frames | Integer. Range: 1 - 10 Default: 1 pocketsphinx.set_property('maxhmmpf', 3000) # Maximum number of HMMs searched per frame | Integer. Range: 1 - 100000 Default: 30000 pocketsphinx.set_property('bestpath', True) # Enable Graph Search | Boolean. Default: true # pocketsphinx.set_property('maxwpf', -1) # pocketsphinx.set_property('maxwpf', 20) # Maximum number of words searched per frame | Range: 1 - 100000 Default: -1 self.elements_stack.append(pocketsphinx) capsfilter_queue = pipeline.get_by_name('capsfilter_queue') capsfilter_queue.set_property('leaky', True) # prefer fresh data capsfilter_queue.set_property('silent', False) capsfilter_queue.set_property('max-size-time', 0) # 0 seconds capsfilter_queue.set_property('max-size-buffers', 0) capsfilter_queue.set_property('max-size-bytes', 0) self.capsfilter_queue_overrun_handler = capsfilter_queue.connect('overrun', self._log_queue_overrun) # capsfilter_queue.connect('overrun', self._on_overrun) # capsfilter_queue.connect('underrun', self._on_underrun) # capsfilter_queue.connect('pushing', self._on_pushing) # capsfilter_queue.connect('running', self._on_running) self.elements_stack.append(capsfilter_queue) ident = pipeline.get_by_name('ident') # ident.connect('handoff', self._on_handoff) self.elements_stack.append(ident) logger.debug("After all self.elements_stack.append() calls") # Set up the queue for data and run the main thread. self.queue = queue.Queue(QUEUE_SIZE) self.thread = get_loop_thread() def _on_handoff(self, element, buf): logger.debug('buf:') pp.pprint(buf) pp.pprint(dir(buf)) logger.debug('on_handoff - %d bytes' % len(buf)) # print 'buf =', buf # print 'dir(buf) =', dir(buf) if self.signed is None: # only ever one caps struct on our buffers struct = buf.get_caps().get_structure(0) # I think these are always set too, but catch just in case try: self.signed = struct["signed"] self.depth = struct["depth"] self.rate = struct["rate"] self.channels = struct["channels"] except: logger.debug('on_handoff: missing caps') pass # raw = str(buf) # # # print 'len(raw) =', len(raw) # # sm = 0 # for i in range(0, len(raw)): # sm += ord(raw[i]) # # print sm # FIXEME: Add somthing like analyse.py # SOURCE: https://github.com/jcupitt/huebert/blob/master/huebert/audio.py def _on_state_changed(self, bus, msg): states = msg.parse_state_changed() # To state is PLAYING if msg.src.get_name() == "pipeline0" and states[1] == 4: logger.info('Inside pipeline0 on _on_state_changed') logger.info("State: {}".format(states[1])) self.ready_sem.release() return False else: # logger.error('NOTHING RETURNED in _on_state_changed') logger.info("State: {}".format(states[1])) def _on_overrun(self, element): logging.debug('on_overrun') def _on_underrun(self, element): logging.debug('on_underrun') def _on_running(self, element): logging.debug('on_running') def _on_pushing(self, element): logging.debug('on_pushing') def _notify_caps(self, pad, args): """The callback for the sinkpad's "notify::caps" signal. """ # The sink has started to receive data, so the stream is ready. # This also is our opportunity to read information about the # stream. self.got_caps = True # Allow constructor to complete. self.ready_sem.release() _got_a_pad = False def _log_queue_overrun(self, queue): cbuffers = queue.get_property('current-level-buffers') cbytes = queue.get_property('current-level-bytes') ctime = queue.get_property('current-level-time') def _new_sample(self, sink): """The callback for appsink's "new-sample" signal. """ if self.running: # New data is available from the pipeline! Dump it into our # queue (or possibly block if we're full). buf = sink.emit('pull-sample').get_buffer() self.queue.put(buf.extract_dup(0, buf.get_size())) return Gst.FlowReturn.OK def _on_message(self, bus, message): """The callback for GstBus's "message" signal (for two kinds of messages). """ # logger.debug("[_on_message](%s, %s)", bus, message) if not self.finished: struct = message.get_structure() if message.type == Gst.MessageType.EOS: # The file is done. Tell the consumer thread. self.queue.put(SENTINEL) if not self.got_caps: logger.error( "If the stream ends before _notify_caps was called, this is an invalid stream.") # If the stream ends before _notify_caps was called, this # is an invalid file. self.read_exc = generator_utils.NoStreamError() self.ready_sem.release() elif struct and struct.get_name() == 'pocketsphinx': if struct['final']: logger.info(struct['hypothesis']) if self.kw_found == 1: # If keyword is set AND qualifier # then perform action self.run_cmd(struct['hypothesis']) else: # If it's the main keyword, # set values wait for qualifier self.result(struct['hypothesis']) elif message.type == Gst.MessageType.ERROR: gerror, debug = message.parse_error() pp.pprint(("gerror,debug:", gerror, debug)) if 'not-linked' in debug: logger.error('not-linked') self.read_exc = generator_utils.NoStreamError() elif 'No such device' in debug: logger.error('No such device') self.read_exc = generator_utils.NoStreamError() else: logger.info("FileReadError") pp.pprint(("SOME FileReadError", bus, message, struct, struct.get_name())) self.read_exc = generator_utils.FileReadError(debug) self.ready_sem.release() # Cleanup. def close(self, force=False): """Close the file and clean up associated resources. Calling `close()` a second time has no effect. """ if self.running or force: self.running = False self.finished = True try: gst_bus = self.gst_bus_stack[0] except: logger.error("Failed to get gst_bus from gst_bus_stack[0]") pass if gst_bus: gst_bus.remove_signal_watch() if self.bus_message_element_handler_id: gst_bus.disconnect(self.bus_message_element_handler_id) if self.bus_message_error_handler_id: gst_bus.disconnect(self.bus_message_error_handler_id) if self.bus_message_eos_handler_id: gst_bus.disconnect(self.bus_message_eos_handler_id) if self.bus_message_state_changed_handler_id: gst_bus.disconnect(self.bus_message_state_changed_handler_id) self.bus = None self.pipeline = None self.codec = None self.bitrate = -1 self.state = None # Unregister for signals, which we registered for above with # `add_signal_watch`. (Without this, GStreamer leaks file # descriptors.) logger.debug('BEFORE p = self.pipelines_stack[0]') p = self.pipelines_stack[0] p.get_bus().remove_signal_watch() logger.debug('AFTER p.get_bus().remove_signal_watch()') # Block spurious signals. appsink = self.elements_stack[0] appsink.get_static_pad("sink").disconnect(self.caps_handler) # Make space in the output queue to let the decoder thread # finish. (Otherwise, the thread blocks on its enqueue and # the interpreter hangs.) try: self.queue.get_nowait() except queue.Empty: pass # Halt the pipeline (closing file). self.stop() # Delete the pipeline object. This seems to be necessary on Python # 2, but not Python 3 for some reason: on 3.5, at least, the # pipeline gets dereferenced automatically. del p def __del__(self): self.close() def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False
loop = GLib.MainLoop() recieved_signals = [] from pydbus import SessionBus bus = SessionBus() ss = bus.get("org.scarlett", object_path="/org/scarlett/Listener") time.sleep(0.5) # taken from tasker ss_failed_signal = bus.subscribe( sender=None, iface="org.scarlett.Listener", signal="SttFailedSignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=catchall_handler, ) ss_rdy_signal = bus.subscribe( sender=None, iface="org.scarlett.Listener", signal="ListenerReadySignal", object="/org/scarlett/Listener", arg0=None, flags=0, signal_fired=catchall_handler, )
Duration = rospy.Duration(4.0) endtime = beginTime+Duration while rospy.Time.now() < endtime: if(field == "TripStart"): pubStart.publish(topicValue) rate.sleep() else: pubResume.publish(topicValue) rate.sleep() pubStartFalse.publish(0) pubResumeFalse.publish(0) if __name__ == "__main__": bus.subscribe(object=dbus_, signal_fired=response_recieved) loop.run() class Client(): def __init__(self): bus = dbus.SessionBus() service = bus.get_object('com.example.service', "/com/example/service") self._message = service.get_dbus_method('get_message', 'com.example.service.Message') self._quit = service.get_dbus_method('quit', 'com.example.service.Quit') def run(self): print "Mesage from service:", self._message() self._quit() if __name__ == "__main__": Client().run()