def __init__(self, lisa_client, botname):
# Init thread class
threading.Thread.__init__(self)
self._stopevent = threading.Event()
configuration = ConfigManagerSingleton.get().getConfiguration()
self.botname = lisa_client.botname.lower()
self.scores = []
self.record_time_start = 0
self.record_time_end = 0
self.wit_thread = None
self.loop = None
if configuration.has_key("keyword_score"):
self.keyword_score = configuration['keyword_score']
else:
self.keyword_score = -10000
# Build Gstreamer pipeline : mic->Pulse->tee|->queue->audioConvert->audioResample->vader->pocketsphinx->fakesink
# |->queue->audioConvert->audioResample->lamemp3enc->appsink
# fakesink : async=false is mandatory for parallelism
self.pipeline = gst.parse_launch('pulsesrc '
#+ '! ladspa-gate Threshold=-30.0 Decay=2.0 Hold=2.0 Attack=0.1 '
+ '! tee name=audio_tee '
+ ' audio_tee. ! queue '
+ ' ! audioconvert ! audioresample '
+ ' ! audio/x-raw-int, format=(string)S16_LE, channels=1, rate=16000 '
+ ' ! lamemp3enc bitrate=64 mono=true '
+ ' ! appsink name=rec_sink emit-signals=true '
+ ' audio_tee. ! queue '
+ ' ! audioconvert ! audioresample '
+ ' ! vader name=vad_asr auto-threshold=true'
+ ' ! tee name=asr_tee '
+ ' asr_tee. ! fakesink async=false name=asr_sink'
+ ' asr_tee. ! pocketsphinx name=asr '
+ ' ! fakesink async=false'
)
# Create recorder
self.recorder = Recorder(lisa_client = lisa_client, listener = self)
# Find client path
if os.path.isdir('/var/lib/lisa/client/pocketsphinx'):
client_path = '/var/lib/lisa/client/pocketsphinx'
else:
client_path = "%s/pocketsphinx" % PWD
# PocketSphinx configuration
asr = self.pipeline.get_by_name('asr')
asr.set_property("dict", "%s/%s.dic" % (client_path, self.botname))
asr.set_property("lm", "%s/%s.lm" % (client_path, self.botname))
if configuration.has_key("hmm"):
hmm_path = "%s/%s" % (client_path, configuration["hmm"])
if os.path.isdir(hmm_path):
asr.set_property("hmm", hmm_path)
asr.connect('result', self._asr_result)
asr.set_property('configured', True)
self.ps = pocketsphinx.Decoder(boxed=asr.get_property('decoder'))
# Start thread
self.start()
def __init__(self, factory):
# Init thread class
threading.Thread.__init__(self)
self._stopevent = threading.Event()
self.configuration = ConfigManager.getConfiguration()
self._ = self.configuration['trans']
self.factory = factory
self.botname = ""
self.scores = []
self.recorder = Recorder(factory = factory)
self.running_state = False
# Find client path
if os.path.isdir('/var/lib/lisa/client/pocketsphinx'):
self.client_path = '/var/lib/lisa/client/pocketsphinx'
else:
self.client_path = "{0}/lib/pocketsphinx".format(self.configuration['path'])
# Initialize with a default bot name
self.setBotName("neo")
# Start thread
threading.Thread.start(self)
class Listener(threading.Thread):
"""
The goal is to listen for a keyword, then it starts a voice recording
"""
#-----------------------------------------------------------------------------
def __init__(self, factory):
# Init thread class
threading.Thread.__init__(self)
self._stopevent = threading.Event()
self.configuration = ConfigManager.getConfiguration()
self._ = self.configuration['trans']
self.factory = factory
self.botname = ""
self.scores = []
self.recorder = Recorder(factory = factory)
self.running_state = False
# Find client path
if os.path.isdir('/var/lib/lisa/client/pocketsphinx'):
self.client_path = '/var/lib/lisa/client/pocketsphinx'
else:
self.client_path = "{0}/lib/pocketsphinx".format(self.configuration['path'])
# Initialize with a default bot name
self.setBotName("neo")
# Start thread
threading.Thread.start(self)
#-----------------------------------------------------------------------------
def setBotName(self, botname):
# If nothing to do
if botname == self.botname:
return
self.botname = botname
# Pause thread
self.setRunningState(state = False)
# Init pipes
self.pipes = []
for i in range(NUM_PIPES):
self.pipes.append({'vad' : None, 'ps' : None, 'timeout' : 0})
# Build Gstreamer pipeline
if self.configuration['asr'] == "ispeech":
enc_str = 'speexenc mode=2'
elif self.configuration['asr'] == "google":
enc_str = 'flacenc'
# Default Wit
else:
enc_str = 'lamemp3enc bitrate=16 mono=true'
pipeline = 'pulsesrc' \
+ ' ! tee name=audio_tee' \
+ ' audio_tee.' \
+ ' ! queue ! audiodynamic characteristics=soft-knee mode=compressor threshold=0.5 ratio=0.5 ! audioconvert ! audioresample' \
+ ' ! audio/x-raw-int, format=(string)S16_LE, channels=1, rate=16000' \
+ ' ! ' + enc_str \
+ ' ! appsink name=rec_sink emit-signals=true async=false' \
+ ' audio_tee.' \
+ ' ! queue ! audiocheblimit mode=1 cutoff=150' \
+ ' ! audiodynamic ! audioconvert ! audioresample' \
+ ' ! tee name=asr_tee'
# Add pocketsphinx
for i in range(NUM_PIPES):
pipeline = pipeline \
+ ' asr_tee.' \
+ ' ! vader name=vad_{0} auto-threshold=true'.format(i) \
+ ' ! pocketsphinx name=asr_{0}'.format(i) \
+ ' ! fakesink async=false'
# Create pipeline
self.pipeline = gst.parse_launch(pipeline)
# Configure pipes
for i in range(NUM_PIPES):
# Initialize vader
vader = self.pipeline.get_by_name('vad_{0}'.format(i))
vader.connect('vader-start', self._vaderStart, i)
vader.connect('vader-stop', self._vaderStop, i)
self.pipes[i]['vad'] = vader
# Initialize pocketsphinx
asr = self.pipeline.get_by_name('asr_{0}'.format(i))
asr.set_property("dict", "{path}/{bot}.dic".format(path = self.client_path, bot = self.botname))
asr.set_property("lm", "{path}/{bot}.lm".format(path = self.client_path, bot = self.botname))
if self.configuration.has_key("hmm"):
if os.path.isdir(self.configuration["hmm"]):
asr.set_property("hmm", self.configuration["hmm"])
else:
hmm_path = "{path}/{hmm}".format(path = self.client_path, hmm = self.configuration["hmm"])
if os.path.isdir(hmm_path):
asr.set_property("hmm", hmm_path)
asr.connect('result', self._asrResult, i)
asr.set_property('configured', 1)
self.pipes[i]['ps'] = pocketsphinx.Decoder(boxed = asr.get_property('decoder'))
# Start pipeline
self.pipeline.set_state(gst.STATE_PLAYING)
# Restart
self.setRunningState(state = True)
#-----------------------------------------------------------------------------
def setRunningState(self, state):
if state == True:
# Restart
self.running_state = True
self.recorder.setRunningState(state = True, rec_sink = self.pipeline.get_by_name('rec_sink'))
else:
# Pause thread
self.running_state = False
self.recorder.setRunningState(state = False)
#-----------------------------------------------------------------------------
def setContinuousMode(self, enabled, wit_context = None):
# Change continuous mode in recorder
self.recorder.setContinuousMode(enabled = enabled, wit_context = wit_context)
#-----------------------------------------------------------------------------
def run(self):
"""
Listener main loop
"""
# Thread loop
while not self._stopevent.isSet():
if self.running_state == True:
for p in self.pipes:
if p['timeout'] > 0 and time() >= p['timeout']:
# Force silent to cut current utterance
p['vad'].set_property('silent', True)
p['vad'].set_property('silent', False)
# Manual start (vader_start may be not called after the forced silence)
p['timeout'] += VADER_MAX_LENGTH
self.recorder.vaderStart()
sleep(.1)
# Stop pipeline
self.pipeline.set_state(gst.STATE_NULL)
self.pipeline = None
#-----------------------------------------------------------------------------
def stop(self):
"""
Stop listener.
"""
# Stop everything
self.recorder.stop()
self._stopevent.set()
#-----------------------------------------------------------------------------
def _vaderStart(self, ob, message, pipe_id):
"""
Vader start detection
"""
# Vader start
if self.pipes[pipe_id]['timeout'] == 0:
self.pipes[pipe_id]['timeout'] = time() + VADER_MAX_LENGTH * (1 + pipe_id / 2.0)
self.recorder.vaderStart()
#-----------------------------------------------------------------------------
def _vaderStop(self, ob, message, pipe_id):
"""
Vader stop detection
"""
# Vader stop
self.pipes[pipe_id]['timeout'] = 0
self.recorder.vaderStop()
#-----------------------------------------------------------------------------
def _asrResult(self, asr, text, uttid, pipe_id):
"""
Result from pocketsphinx : checking keyword recognition
"""
# When not running
if self.running_state == False:
return
# Get score from decoder
dec_text, dec_uttid, dec_score = self.pipes[pipe_id]['ps'].get_hyp()
# Detection must have a minimal score to be valid
if dec_score != 0 and dec_score < self.configuration['keyword_score']:
log.msg("I recognized the {word} keyword but I think it's a false positive according the {score} score".format(word = self.botname, score = dec_score))
return
# Activate recorder
self.recorder.activate()
# Logs
self.scores.append(dec_score)
log.msg("======================")
log.msg("{word} keyword detected".format(word = self.botname))
log.msg("score: {score} (min {min}, moy {moy}, max {max})".format(score = dec_score, min = min(self.scores), moy = sum(self.scores) / len(self.scores), max = max(self.scores)))
class Listener(threading.Thread):
"""
The goal is to listen for a keyword, then it starts a voice recording
"""
def __init__(self, lisa_client, botname):
# Init thread class
threading.Thread.__init__(self)
self._stopevent = threading.Event()
configuration = ConfigManagerSingleton.get().getConfiguration()
self.botname = lisa_client.botname.lower()
self.scores = []
self.record_time_start = 0
self.record_time_end = 0
self.wit_thread = None
self.loop = None
if configuration.has_key("keyword_score"):
self.keyword_score = configuration['keyword_score']
else:
self.keyword_score = -10000
# Build Gstreamer pipeline : mic->Pulse->tee|->queue->audioConvert->audioResample->vader->pocketsphinx->fakesink
# |->queue->audioConvert->audioResample->lamemp3enc->appsink
# fakesink : async=false is mandatory for parallelism
self.pipeline = gst.parse_launch('pulsesrc '
#+ '! ladspa-gate Threshold=-30.0 Decay=2.0 Hold=2.0 Attack=0.1 '
+ '! tee name=audio_tee '
+ ' audio_tee. ! queue '
+ ' ! audioconvert ! audioresample '
+ ' ! audio/x-raw-int, format=(string)S16_LE, channels=1, rate=16000 '
+ ' ! lamemp3enc bitrate=64 mono=true '
+ ' ! appsink name=rec_sink emit-signals=true '
+ ' audio_tee. ! queue '
+ ' ! audioconvert ! audioresample '
+ ' ! vader name=vad_asr auto-threshold=true'
+ ' ! tee name=asr_tee '
+ ' asr_tee. ! fakesink async=false name=asr_sink'
+ ' asr_tee. ! pocketsphinx name=asr '
+ ' ! fakesink async=false'
)
# Create recorder
self.recorder = Recorder(lisa_client = lisa_client, listener = self)
# Find client path
if os.path.isdir('/var/lib/lisa/client/pocketsphinx'):
client_path = '/var/lib/lisa/client/pocketsphinx'
else:
client_path = "%s/pocketsphinx" % PWD
# PocketSphinx configuration
asr = self.pipeline.get_by_name('asr')
asr.set_property("dict", "%s/%s.dic" % (client_path, self.botname))
asr.set_property("lm", "%s/%s.lm" % (client_path, self.botname))
if configuration.has_key("hmm"):
hmm_path = "%s/%s" % (client_path, configuration["hmm"])
if os.path.isdir(hmm_path):
asr.set_property("hmm", hmm_path)
asr.connect('result', self._asr_result)
asr.set_property('configured', True)
self.ps = pocketsphinx.Decoder(boxed=asr.get_property('decoder'))
# Start thread
self.start()
def run(self):
"""
Listener main loop
"""
Speaker.speak("ready")
self.pipeline.set_state(gst.STATE_PLAYING)
# Thread loop
self.loop = gobject.MainLoop()
self.loop.run()
def stop(self):
"""
Stop listener.
"""
Speaker.speak('lost_server')
# Stop everything
self.pipeline.set_state(gst.STATE_NULL)
self.recorder.stop()
if self.loop is not None:
self.loop.quit()
def _asr_result(self, asr, text, uttid):
"""
Result from pocketsphinx : checking keyword recognition
"""
# Check keyword detection
if text.lower() == self.botname and self.recorder.get_running_state() == False:
# Get scrore from decoder
dec_text, dec_uttid, dec_score = self.ps.get_hyp()
# Detection must have a minimal score to be valid
if dec_score < self.keyword_score:
log.msg("I recognized the %s keyword but I think it's a false positive according the %s score" % (self.botname, dec_score))
return
# Logs
self.scores.append(dec_score)
log.msg("======================")
log.msg("%s keyword detected" % self.botname)
log.msg("score: {} (min {}, moy {}, max {})".format(dec_score, min(self.scores), sum(self.scores) / len(self.scores), max(self.scores)))
# Start voice recording
Speaker.speak('yes')
# Start recorder
self.recorder.set_running_state(True)
def get_pipeline(self):
"""
Return Gstreamer pipeline
"""
return self.pipeline