def read_write_audio(self, p, stream, wf, play_buffer):
"""Send some of the available data to the output.
It should be a non-blocking operation.
Therefore:
1) do not send more then play_buffer_frames
2) send only if stream.get_write_available() is more then the frame size
"""
if self.audio_play.poll():
while self.audio_play.poll() \
and len(play_buffer) < self.cfg['AudioIO']['play_buffer_size'] \
and stream.get_write_available() > self.cfg['Audio']['samples_per_frame']:
# send to play frames from input
data_play = self.audio_play.recv()
if isinstance(data_play, Frame):
stream.write(data_play.payload)
play_buffer.append(data_play)
if self.cfg['AudioIO']['debug']:
print '.',
sys.stdout.flush()
elif isinstance(data_play, Command):
if data_play.parsed['__name__'] == 'utterance_start':
self.commands.send(Command('play_utterance_start()', 'AudioIO', 'HUB'))
if data_play.parsed['__name__'] == 'utterance_end':
self.commands.send(Command('play_utterance_end()', 'AudioIO', 'HUB'))
def process_da(self, da):
if da != "silence()":
text = self.nlg.generate(da)
if self.cfg['NLG']['debug']:
s = []
s.append("NLG Output")
s.append("-"*60)
s.append(text)
s.append("")
s = '\n'.join(s)
self.cfg['Logging']['system_logger'].debug(s)
self.cfg['Logging']['session_logger'].text("system", text)
self.commands.send(Command('nlg_text_generated()', 'NLG', 'HUB'))
self.commands.send(TTSText(text))
self.text_out.send(TTSText(text))
else:
# the input dialogue is silence. Therefore, do not generate eny output.
if self.cfg['NLG']['debug']:
s = []
s.append("NLG Output")
s.append("-"*60)
s.append("DM sent 'silence()' therefore generating nothing")
s.append("")
s = '\n'.join(s)
self.cfg['Logging']['system_logger'].debug(s)
self.cfg['Logging']['session_logger'].text("system", "_silence_")
self.commands.send(Command('nlg_text_generated()', 'NLG', 'HUB'))
def read_write_audio(self):
"""Send as much possible of the available data to the output and read as much as possible from the input.
It should be a non-blocking operation.
"""
if (self.local_audio_play
and (self.mem_player.get_write_available() >
self.cfg['Audio']['samples_per_frame'] * 2)):
# send a frame from input to be played
data_play = self.local_audio_play.popleft()
if self.audio_playing and isinstance(data_play, Frame):
if len(data_play
) == self.cfg['Audio']['samples_per_frame'] * 2:
self.last_frame_id = self.mem_player.put_frame(
data_play.payload)
self.cfg['Logging']['session_logger'].rec_write(
self.audio_playing, data_play.payload)
elif isinstance(data_play, Command):
if data_play.parsed['__name__'] == 'utterance_start':
self.audio_playing = data_play.parsed['fname']
self.message_queue.append((Command(
'play_utterance_start(user_id="{uid}",fname="{fname}")'
.format(uid=data_play.parsed['user_id'],
fname=data_play.parsed['fname']), 'VoipIO',
'HUB'), self.last_frame_id))
try:
if data_play.parsed['log'] == "true":
self.cfg['Logging']['session_logger'].rec_start(
"system", data_play.parsed['fname'])
except SessionLoggerException as e:
self.cfg['Logging']['system_logger'].exception(e)
if self.audio_playing and data_play.parsed[
'__name__'] == 'utterance_end':
self.audio_playing = None
self.message_queue.append((Command(
'play_utterance_end(user_id="{uid}",fname="{fname})'.
format(uid=data_play.parsed['user_id'],
fname=data_play.parsed['fname']), 'VoipIO',
'HUB'), self.last_frame_id))
try:
if data_play.parsed['log'] == "true":
self.cfg['Logging']['session_logger'].rec_end(
data_play.parsed['fname'])
except SessionLoggerException as e:
self.cfg['Logging']['system_logger'].exception(e)
if (self.mem_capture.get_read_available() >
self.cfg['Audio']['samples_per_frame'] * 2):
# Get and send recorded data, it must be read at the other end.
data_rec = self.mem_capture.get_frame()
# send the audio only if the call is connected
# ignore any audio signal left after the call was disconnected
if self.audio_recording:
self.audio_record.send(Frame(data_rec))
def read_write_audio(self):
# read input audio
if self.local_audio_in:
if len(self.local_audio_in) > 10:
print "VAD unprocessed frames:", len(self.local_audio_in)
# read recorded audio
data_rec = self.local_audio_in.popleft()
if isinstance(data_rec, Frame):
# buffer the recorded and played audio
self.deque_audio_in.append(data_rec)
decision = self.vad.decide(data_rec.payload)
vad, change = self.smoothe_decison(decision)
if self.cfg['VAD']['debug']:
self.system_logger.debug("vad: %s change: %s" % (vad, change))
if vad:
self.system_logger.debug('+')
else:
self.system_logger.debug('-')
if change == 'speech':
# Create new wave file.
timestamp = datetime.now().strftime('%Y-%m-%d--%H-%M-%S.%f')
self.vad_fname = 'vad-{stamp}.wav'.format(stamp=timestamp)
self.session_logger.turn("user")
self.session_logger.rec_start("user", self.vad_fname)
# Inform both the parent and the consumer.
self.audio_out.send(Command('speech_start(fname="%s")' % self.vad_fname, 'VAD', 'AudioIn'))
self.commands.send(Command('speech_start(fname="%s")' % self.vad_fname, 'VAD', 'HUB'))
elif change == 'non-speech':
self.session_logger.rec_end(self.vad_fname)
# Inform both the parent and the consumer.
self.audio_out.send(Command('speech_end(fname="%s")' % self.vad_fname, 'VAD', 'AudioIn'))
self.commands.send(Command('speech_end(fname="%s")' % self.vad_fname, 'VAD', 'HUB'))
if vad:
while self.deque_audio_in:
# Send or save all potentially queued data.
# - When there is change to speech, there will be
# several frames of audio;
# - If there is no change, then there will be only
# one queued frame.
data_rec = self.deque_audio_in.popleft()
# Send the result.
self.audio_out.send(data_rec)
self.session_logger.rec_write(self.vad_fname, data_rec)
def read_slu_hypotheses_write_dialogue_act(self):
# read SLU hypothesis
if self.slu_hypotheses_in.poll():
# read SLU hypothesis
data_slu = self.slu_hypotheses_in.recv()
if self.epilogue_state:
# we have got another turn, now we can hang up.
self.cfg['Logging']['session_logger'].turn("system")
self.dm.log_state()
self.cfg['Logging']['session_logger'].dialogue_act(
"system", self.epilogue_da)
self.commands.send(DMDA(self.epilogue_da, 'DM', 'HUB'))
self.commands.send(Command('hangup()', 'DM', 'HUB'))
elif isinstance(data_slu, SLUHyp):
# reset measuring of the user silence
self.last_user_da_time = time.time()
self.last_user_diff_time = time.time()
# process the input DA
self.dm.da_in(data_slu.hyp, utterance=data_slu.asr_hyp)
self.cfg['Logging']['session_logger'].turn("system")
self.dm.log_state()
da = self.dm.da_out()
# do not communicate directly with the NLG, let the HUB decide
# to do work. The generation of the output must by synchronised with the input.
if da.has_dat("bye"):
self.epilogue_state = self.epilogue()
self.epilogue_da = da
if not self.epilogue_state:
self.cfg['Logging']['session_logger'].dialogue_act(
"system", da)
self.commands.send(DMDA(da, 'DM', 'HUB'))
self.commands.send(Command('hangup()', 'DM', 'HUB'))
else:
if self.cfg['DM']['debug']:
s = []
s.append("DM Output")
s.append("-" * 60)
s.append(unicode(da))
s.append("")
s = '\n'.join(s)
self.cfg['Logging']['system_logger'].debug(s)
self.cfg['Logging']['session_logger'].dialogue_act(
"system", da)
self.commands.send(DMDA(da, 'DM', 'HUB'))
elif isinstance(data_slu, Command):
self.cfg['Logging']['system_logger'].info(data_slu)
else:
raise DMException('Unsupported input.')
def make_call(self, remote_uri):
""" Call provided URI. Check whether it is allowed.
"""
# *WARNING* pjsip only handles standard string, not UNICODE !
remote_uri = str(remote_uri)
try:
uri = remote_uri
if is_phone_number_hash(uri):
uri = recover_phone_number_from_hash(uri)
remote_uri = self.normalise_uri(remote_uri)
uri = self.normalise_uri(uri)
if self.cfg['VoipIO']['debug']:
print "-" * 120
print " Remote uri: ", remote_uri
print " Making a call to:", uri
print "-" * 120
print
if self.is_sip_uri(uri):
# create a call back for the call
call_cb = CallCallback(self.cfg, None, self)
self.call = self.acc.make_call(uri, cb=call_cb)
# send a message that we are calling to uri
self.commands.send(
Command(
'make_call(remote_uri="%s")' %
get_user_from_uri(remote_uri), 'VoipIO', 'HUB'))
return self.call
elif uri == "blocked":
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(
'VoipIO.make_call: Calling a blocked phone number - %s'
% uri)
# send a message that the provided uri is blocked
self.commands.send(
Command('blocked_uri(remote_uri="%s")' % remote_uri,
'VoipIO', 'HUB'))
else:
self.cfg['Logging']['system_logger'].error(
'VoipIO.make_call: Calling SIP URI which is not recognised as valid SIP URI - %s'
% uri)
# send a message that the provided uri is invalid
self.commands.send(
Command('invalid_uri(remote_uri="%s")' % remote_uri,
'VoipIO', 'HUB'))
except pj.Error as e:
print "Exception: " + unicode(e)
return None
def process_pending_commands(self):
"""Process all pending commands.
Available commands:
stop() - stop processing and exit the process
flush() - flush input buffers.
Now it only flushes the input connection.
Returns True iff the process should terminate.
"""
while self.local_commands:
command = self.local_commands.popleft()
if self.cfg['ASR']['debug']:
self.system_logger.debug(command)
if isinstance(command, Command):
if command.parsed['__name__'] == 'stop':
return True
if command.parsed['__name__'] == 'flush':
# Discard all data in input buffers.
while self.audio_in.poll():
self.audio_in.recv()
self.local_audio_in.clear()
self.asr.flush()
self.recognition_on = False
self.commands.send(Command("flushed()", 'ASR', 'HUB'))
return False
return False
def process_pending_commands(self):
"""Process all pending commands.
Available commands:
stop() - stop processing and exit the process
flush() - flush input buffers.
Now it only flushes the input connection.
Return True if the process should terminate.
"""
while self.commands.poll():
command = self.commands.recv()
if self.cfg['NLG']['debug']:
self.cfg['Logging']['system_logger'].debug(command)
if isinstance(command, Command):
if command.parsed['__name__'] == 'stop':
return True
if command.parsed['__name__'] == 'flush':
# discard all data in in input buffers
while self.dialogue_act_in.poll():
data_in = self.dialogue_act_in.recv()
# the NLG component does not have to be flushed
#self.nlg.flush()
self.commands.send(Command("flushed()", 'NLG', 'HUB'))
return False
elif isinstance(command, DMDA):
self.process_da(command.da)
return False
def play_intro(cfg, tts_commands, intro_id, last_intro_id):
for i in range(len(cfg['RepeatAfterMe']['introduction'])):
last_intro_id = str(intro_id)
intro_id += 1
tts_commands.send(Command('synthesize(user_id="%s",text="%s")' % (last_intro_id, cfg['RepeatAfterMe']['introduction'][i]), 'HUB', 'TTS'))
return intro_id, last_intro_id
def on_dtmf_digit(self, digits):
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug("VoipIO::on_dtmf_digit")
# send a message that a digit was recieved
self.commands.send(
Command('dtmf_digit(digit="%s")' % digits, 'VoipIO', 'HUB'))
def play_intro(cfg, tts_commands, intro_id, last_intro_id):
cfg['Logging']['session_logger'].turn("system")
for i in range(len(cfg['Switchboard']['introduction'])):
last_intro_id = str(intro_id)
intro_id += 1
tts_commands.send(Command('synthesize(user_id="%s",text="%s",log="true")' % (last_intro_id, cfg['Switchboard']['introduction'][i]), 'HUB', 'TTS'))
return intro_id, last_intro_id
def synthesize(self, user_id, text, log="true"):
if text == "_silence_" or text == "silence()":
# just let the TTS generate an empty wav
text == ""
wav = []
timestamp = datetime.now().strftime('%Y-%m-%d--%H-%M-%S.%f')
fname = 'tts-{stamp}.wav'.format(stamp=timestamp)
self.commands.send(
Command(
'tts_start(user_id="%s",text="%s",fname="%s")' %
(user_id, text, fname), 'TTS', 'HUB'))
self.audio_out.send(
Command(
'utterance_start(user_id="%s",text="%s",fname="%s",log="%s")' %
(user_id, text, fname, log), 'TTS', 'AudioOut'))
segments = self.parse_into_segments(text)
for i, segment_text in enumerate(segments):
segment_wav = self.tts.synthesize(segment_text)
segment_wav = self.remove_start_and_final_silence(segment_wav)
if i < len(segments) - 1:
# add silence only for non-final segments
segment_wav += self.gen_silence()
wav.append(segment_wav)
segment_wav = various.split_to_bins(
segment_wav, 2 * self.cfg['Audio']['samples_per_frame'])
for frame in segment_wav:
self.audio_out.send(Frame(frame))
self.commands.send(
Command(
'tts_end(user_id="%s",text="%s",fname="%s")' %
(user_id, text, fname), 'TTS', 'HUB'))
self.audio_out.send(
Command(
'utterance_end(user_id="%s",text="%s",fname="%s",log="%s")' %
(user_id, text, fname, log), 'TTS', 'AudioOut'))
def on_rejected_call(self, remote_uri):
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(
"VoipIO::on_rejected_call - from %s" % remote_uri)
# send a message that we rejected an incoming call
self.commands.send(
Command(
'rejected_call(remote_uri="%s")' %
get_user_from_uri(remote_uri), 'VoipIO', 'HUB'))
def on_call_connecting(self, remote_uri):
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(
"VoipIO::on_call_connecting")
# send a message that the call is connecting
self.commands.send(
Command(
'call_connecting(remote_uri="%s")' %
get_user_from_uri(remote_uri), 'VoipIO', 'HUB'))
def _send_play_cmd(self, which, utt_id):
if not utt_id in self.utt_info:
return False
else:
data = self.utt_info[utt_id]
cmd = Command(
'play_utterance_{which}(user_id="{uid}",fname="{fname})'.
format(which=which, uid=data['user_id'],
fname=data['fname']), 'VoipIO', 'HUB')
self.commands.send(cmd)
return True
def on_incoming_call(self, remote_uri):
""" Signals an incoming call.
"""
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(
"VoipIO::on_incoming_call - from %s" % remote_uri)
# send a message that there is a new incoming call
self.commands.send(
Command(
'incoming_call(remote_uri="%s")' %
get_user_from_uri(remote_uri), 'VoipIO', 'HUB'))
def on_call_disconnected(self, remote_uri, code):
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(
"VoipIO::on_call_disconnected")
# disable recording of audio
self.audio_recording = False
# send a message that the call is disconnected
self.commands.send(
Command(
'call_disconnected(remote_uri="%s", code="%s")' %
(get_user_from_uri(remote_uri), str(code)), 'VoipIO', 'HUB'))
def on_call_confirmed(self, remote_uri):
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(
"VoipIO::on_call_confirmed")
# enable recording of audio
self.audio_recording = True
# send a message that the call is confirmed
self.commands.send(
Command(
'call_confirmed(remote_uri="%s")' %
get_user_from_uri(remote_uri), 'VoipIO', 'HUB'))
def read_asr_hypotheses_write_slu_hypotheses(self):
if self.asr_hypotheses_in.poll():
data_asr = self.asr_hypotheses_in.recv()
if isinstance(data_asr, ASRHyp):
slu_hyp = self.slu.parse(data_asr.hyp)
fname = data_asr.fname
confnet = None
nblist = None
if isinstance(slu_hyp, DialogueActConfusionNetwork):
confnet = slu_hyp
nblist = slu_hyp.get_da_nblist()
elif isinstance(slu_hyp, DialogueActNBList):
confnet = None
nblist = slu_hyp
if self.cfg['SLU']['debug']:
s = []
s.append("SLU Hypothesis")
s.append("-" * 60)
s.append("Confnet:")
s.append(unicode(confnet))
s.append("Nblist:")
s.append(unicode(nblist))
s.append("")
s = '\n'.join(s)
self.cfg['Logging']['system_logger'].debug(s)
self.cfg['Logging']['session_logger'].slu("user", fname, nblist, confnet=confnet)
self.commands.send(Command('slu_parsed(fname="%s")' % fname, 'SLU', 'HUB'))
self.slu_hypotheses_out.send(SLUHyp(slu_hyp, asr_hyp=data_asr.hyp))
elif isinstance(data_asr, Command):
self.cfg['Logging']['system_logger'].info(data_asr)
else:
raise SLUException('Unsupported input.')
def process_pending_commands(self):
"""Process all pending commands.
Available aio_com:
stop() - stop processing and exit the process
flush() - flush input buffers.
Now it only flushes the input connection.
Return True if the process should terminate.
"""
while self.local_commands:
command = self.local_commands.popleft()
#if self.cfg['VAD']['debug']:
self.system_logger.debug(command)
if isinstance(command, Command):
if command.parsed['__name__'] == 'stop':
return True
if command.parsed['__name__'] == 'flush':
# discard all data in in input buffers
while self.audio_in.poll():
data_play = self.audio_in.recv()
self.local_audio_in.clear()
self.detection_window_speech.clear()
self.detection_window_sil.clear()
self.deque_audio_in.clear()
# reset other state variables
self.last_vad = False
self.commands.send(Command("flushed()", 'VAD', 'HUB'))
return False
return False
def process_pending_commands(self):
"""Process all pending commands.
Available aio_com:
stop() - stop processing and exit the process
flush() - flush input buffers.
Now it only flushes the input connection.
Return True if the process should terminate.
"""
while self.commands.poll():
command = self.commands.recv()
if self.cfg['TTS']['debug']:
self.cfg['Logging']['system_logger'].debug(command)
if isinstance(command, Command):
if command.parsed['__name__'] == 'stop':
return True
if command.parsed['__name__'] == 'flush':
# discard all data in in input buffers
while self.text_in.poll():
data_in = self.text_in.recv()
self.commands.send(Command("flushed()", 'TTS', 'HUB'))
return False
if command.parsed['__name__'] == 'synthesize':
self.synthesize(command.parsed['user_id'],
command.parsed['text'],
command.parsed['log'])
return False
return False
def run(self):
try:
cfg = self.cfg
vio_commands, vio_child_commands = multiprocessing.Pipe(
) # used to send commands to VoipIO
vio_record, vio_child_record = multiprocessing.Pipe(
) # I read from this connection recorded audio
vio_play, vio_child_play = multiprocessing.Pipe(
) # I write in audio to be played
vad_commands, vad_child_commands = multiprocessing.Pipe(
) # used to send commands to VAD
vad_audio_out, vad_child_audio_out = multiprocessing.Pipe(
) # used to read output audio from VAD
asr_commands, asr_child_commands = multiprocessing.Pipe(
) # used to send commands to ASR
asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe(
) # used to read ASR hypotheses
slu_commands, slu_child_commands = multiprocessing.Pipe(
) # used to send commands to SLU
slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe(
) # used to read SLU hypotheses
dm_commands, dm_child_commands = multiprocessing.Pipe(
) # used to send commands to DM
dm_actions_out, dm_child_actions = multiprocessing.Pipe(
) # used to read DM actions
nlg_commands, nlg_child_commands = multiprocessing.Pipe(
) # used to send commands to NLG
nlg_text_out, nlg_child_text = multiprocessing.Pipe(
) # used to read NLG output
tts_commands, tts_child_commands = multiprocessing.Pipe(
) # used to send commands to TTS
command_connections = [
vio_commands, vad_commands, asr_commands, slu_commands,
dm_commands, nlg_commands, tts_commands
]
non_command_connections = [
vio_record, vio_child_record, vio_play, vio_child_play,
vad_audio_out, vad_child_audio_out, asr_hypotheses_out,
asr_child_hypotheses, slu_hypotheses_out, slu_child_hypotheses,
dm_actions_out, dm_child_actions, nlg_text_out, nlg_child_text
]
vio = self.voice_io_cls(self.cfg, vio_child_commands,
vio_child_record, vio_child_play,
self.close_event)
vad = VAD(self.cfg, vad_child_commands, vio_record,
vad_child_audio_out, self.close_event)
asr = ASR(self.cfg, asr_child_commands, vad_audio_out,
asr_child_hypotheses, self.close_event)
slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out,
slu_child_hypotheses, self.close_event)
dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out,
dm_child_actions, self.close_event)
nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out,
nlg_child_text, self.close_event)
tts = TTS(self.cfg, tts_child_commands, nlg_text_out, vio_play,
self.close_event)
vio.start()
vad.start()
asr.start()
slu.start()
dm.start()
nlg.start()
tts.start()
self.write_pid_file([['vio', vio.pid], ['vad', vad.pid],
['asr', asr.pid], ['slu', slu.pid],
['dm', dm.pid], ['nlg', nlg.pid],
['tts', tts.pid]])
cfg['Logging']['session_logger'].set_close_event(self.close_event)
cfg['Logging']['session_logger'].set_cfg(cfg)
cfg['Logging']['session_logger'].start()
cfg['Logging']['session_logger'].cancel_join_thread()
# init the system
call_start = 0
call_back_time = -1
call_back_uri = None
number_of_turns = -1
s_voice_activity = False
s_last_voice_activity_time = 0
u_voice_activity = False
u_last_voice_activity_time = 0
s_last_dm_activity_time = 0
u_last_input_timeout = 0
call_connected = False
hangup = False
ncalls = 0
outstanding_nlg_da = None
call_db = CallDB(self.cfg, self.cfg[self.hubname]['call_db'],
self.cfg[self.hubname]['period'])
#call_db.log()
while 1:
# Check the close event.
if self.close_event.is_set():
print 'Received close event in: %s' % multiprocessing.current_process(
).name
return
time.sleep(self.cfg['Hub']['main_loop_sleep_time'])
if call_back_time != -1 and call_back_time < time.time():
vio_commands.send(
Command('make_call(destination="%s")' % call_back_uri,
'HUB', 'VoipIO'))
call_back_time = -1
call_back_uri = None
# read all messages
if vio_commands.poll():
command = vio_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "incoming_call":
self.cfg['Logging']['system_logger'].session_start(
command.parsed['remote_uri'])
self.cfg['Logging'][
'session_logger'].session_start(
self.cfg['Logging']
['system_logger'].get_session_dir_name())
self.cfg['Logging'][
'system_logger'].session_system_log(
'config = ' + unicode(self.cfg))
self.cfg['Logging']['session_logger'].config(
'config = ' + unicode(self.cfg))
self.cfg['Logging']['session_logger'].header(
self.cfg['Logging']["system_name"],
self.cfg['Logging']["version"])
self.cfg['Logging']['session_logger'].input_source(
"voip")
self.cfg['Analytics'].start_session(
command.parsed['remote_uri'])
self.cfg['Analytics'].track_event(
'vhub', 'incoming_call',
command.parsed['remote_uri'])
if command.parsed['__name__'] == "rejected_call":
call_back_time = time.time() + self.cfg['VoipHub'][
'wait_time_before_calling_back']
call_back_uri = command.parsed['remote_uri']
self.cfg['Analytics'].track_event(
'vhub', 'rejected_call',
command.parsed['remote_uri'])
if command.parsed[
'__name__'] == "rejected_call_from_blacklisted_uri":
remote_uri = command.parsed['remote_uri']
num_all_calls, total_time, last_period_num_calls, last_period_total_time, last_period_num_short_calls = call_db.get_uri_stats(
remote_uri)
m = []
m.append('')
m.append('=' * 120)
m.append(
'Rejected incoming call from blacklisted URI: %s'
% remote_uri)
m.append('-' * 120)
m.append('Total calls: %d' %
num_all_calls)
m.append('Total time (min): %0.1f' %
(total_time / 60.0, ))
m.append('Last period short calls: %d' %
last_period_num_short_calls)
m.append('Last period total calls: %d' %
last_period_num_calls)
m.append('Last period total time (min): %0.1f' %
(last_period_total_time / 60.0, ))
m.append('=' * 120)
m.append('')
self.cfg['Logging']['system_logger'].info(
'\n'.join(m))
self.cfg['Analytics'].track_event(
'vhub', 'rejected_call_from_blacklisted_uri',
command.parsed['remote_uri'])
if command.parsed['__name__'] == "call_connecting":
self.cfg['Analytics'].track_event(
'vhub', 'call_connecting',
command.parsed['remote_uri'])
if command.parsed['__name__'] == "call_confirmed":
remote_uri = command.parsed['remote_uri']
num_all_calls, total_time, last_period_num_calls, last_period_total_time, last_period_num_short_calls = call_db.get_uri_stats(
remote_uri)
m = []
m.append('')
m.append('=' * 120)
m.append('Incoming call from : %s' %
remote_uri)
m.append('-' * 120)
m.append('Total calls: %d' %
num_all_calls)
m.append('Total time (min): %0.1f' %
(total_time / 60.0, ))
m.append('Last period short calls: %d' %
last_period_num_short_calls)
m.append('Last period total calls: %d' %
last_period_num_calls)
m.append('Last period total time (min): %0.1f' %
(last_period_total_time / 60.0, ))
m.append('-' * 120)
if last_period_num_calls > self.cfg[self.hubname]['last_period_max_num_calls'] or \
last_period_total_time > self.cfg[self.hubname]['last_period_max_total_time'] or \
last_period_num_short_calls > self.cfg[self.hubname]['last_period_max_num_short_calls'] :
# prepare for ending the call
call_connected = True
call_start = time.time()
number_of_turns = -1
s_voice_activity = True
s_last_voice_activity_time = time.time()
u_voice_activity = False
u_last_voice_activity_time = time.time()
u_last_input_timeout = time.time()
hangup = True
self.cfg['Logging']['session_logger'].turn(
"system")
tts_commands.send(
Command(
'synthesize(text="%s",log="true")' %
self.cfg[self.hubname]
['limit_reached_message'], 'HUB',
'TTS'))
vio_commands.send(
Command(
'black_list(remote_uri="%s",expire="%d")'
% (remote_uri, time.time() + self.cfg[
self.hubname]['blacklist_for']),
'HUB', 'VoipIO'))
m.append('CALL REJECTED')
else:
# init the system
call_connected = True
call_start = time.time()
number_of_turns = 0
s_voice_activity = False
s_last_voice_activity_time = 0
u_voice_activity = False
u_last_voice_activity_time = time.time()
u_last_input_timeout = time.time()
hangup = False
dm_commands.send(
Command('new_dialogue()', 'HUB', 'DM'))
m.append('CALL ACCEPTED')
m.append('=' * 120)
m.append('')
self.cfg['Logging']['system_logger'].info(
'\n'.join(m))
call_db.track_confirmed_call(remote_uri)
self.cfg['Analytics'].track_event(
'vhub', 'call_confirmed',
command.parsed['remote_uri'])
if command.parsed['__name__'] == "call_disconnected":
# flush vio, when flushed, vad will be flushed
vio_commands.send(
Command('flush()', 'HUB', 'VoipIO'))
self.cfg['Logging']['system_logger'].session_end()
self.cfg['Logging']['session_logger'].session_end()
remote_uri = command.parsed['remote_uri']
call_db.track_disconnected_call(remote_uri)
call_connected = False
number_of_turns = -1
ncalls += 1
self.cfg['Analytics'].track_event(
'vhub', 'call_disconnected',
command.parsed['remote_uri'])
dm_commands.send(
Command('prepare_new_dialogue()', 'HUB', 'DM'))
if command.parsed[
'__name__'] == "play_utterance_start":
s_voice_activity = True
s_last_voice_activity_time = time.time()
if command.parsed['__name__'] == "play_utterance_end":
s_voice_activity = False
s_last_voice_activity_time = time.time()
if command.parsed['__name__'] == "flushed":
# flush vad, when flushed, asr will be flushed
vad_commands.send(Command('flush()', 'HUB', 'VAD'))
if command.parsed['__name__'] == "flushed_out":
# process the outstanding DA if necessary
if outstanding_nlg_da:
nlg_commands.send(
DMDA(outstanding_nlg_da, 'HUB', 'NLG'))
outstanding_nlg_da = None
if vad_commands.poll():
command = vad_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "speech_start":
u_voice_activity = True
# interrupt the talking system
# this will be replaced with pausing the system when the necessary extension of pjsip
# is implemented
if s_voice_activity and s_last_voice_activity_time + 0.02 < current_time:
# if the system is still talking then flush the output
self.cfg['Logging']['session_logger'].barge_in(
"system")
# when a user barge in into the output, all the output pipe line
# must be flushed
nlg_commands.send(
Command('flush()', 'HUB', 'NLG'))
s_voice_activity = False
s_last_voice_activity_time = time.time()
if command.parsed['__name__'] == "speech_end":
u_voice_activity = False
u_last_voice_activity_time = time.time()
if command.parsed['__name__'] == "flushed":
# flush asr, when flushed, slu will be flushed
asr_commands.send(Command('flush()', 'HUB', 'ASR'))
if asr_commands.poll():
command = asr_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, ASRHyp):
vio_commands.send(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "flushed":
# flush slu, when flushed, dm will be flushed
slu_commands.send(Command('flush()', 'HUB', 'SLU'))
if slu_commands.poll():
command = slu_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "flushed":
# flush dm, when flushed, nlg will be flushed
dm_commands.send(Command('flush()', 'HUB', 'DM'))
dm_commands.send(
Command('end_dialogue()', 'HUB', 'DM'))
if dm_commands.poll():
command = dm_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "hangup":
# prepare for ending the call
hangup = True
self.cfg['Analytics'].track_event(
'vhub', 'system_hangup')
if command.parsed['__name__'] == "flushed":
# flush nlg, when flushed, tts will be flushed
nlg_commands.send(Command('flush()', 'HUB', 'NLG'))
elif isinstance(command, DMDA):
# record the time of the last system generated dialogue act
s_last_dm_activity_time = time.time()
number_of_turns += 1
if command.da != "silence()":
# if the DM generated non-silence dialogue act, then continue in processing it
if s_voice_activity and s_last_voice_activity_time + 0.02 < current_time:
# if the system is still talking then flush the output
self.cfg['Logging']['session_logger'].barge_in(
"system")
# when a user barge in into the output, all the output pipe line
# must be flushed
nlg_commands.send(
Command('flush()', 'HUB', 'NLG'))
s_voice_activity = False
s_last_voice_activity_time = time.time()
# the DA will be send when all the following components are flushed
outstanding_nlg_da = command.da
else:
nlg_commands.send(
DMDA(command.da, "HUB", "NLG"))
if nlg_commands.poll():
command = nlg_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, TTSText):
vio_commands.send(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "flushed":
# flush tts, when flushed, vio will be flushed
tts_commands.send(Command('flush()', 'HUB', 'TTS'))
if tts_commands.poll():
command = tts_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "flushed":
# flush vio_out
vio_commands.send(
Command('flush_out()', 'HUB', 'VIO'))
current_time = time.time()
s_diff = current_time - s_last_voice_activity_time
u_diff = current_time - u_last_voice_activity_time
t_diff = current_time - u_last_input_timeout
timeout = self.cfg['DM']['input_timeout']
if call_connected and \
not s_voice_activity and not u_voice_activity and \
s_diff > timeout and \
u_diff > timeout and \
t_diff > timeout:
u_last_input_timeout = time.time()
print 'timeout!!!!!', time.time()
dm_commands.send(
Command(
'timeout(silence_time="%0.3f")' %
min(s_diff, u_diff), 'HUB', 'DM'))
if hangup and s_last_dm_activity_time + 2.0 < current_time and \
s_voice_activity == False and s_last_voice_activity_time + 2.0 < current_time:
# we are ready to hangup only when all voice activity is finished
hangup = False
vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))
if number_of_turns != -1 and current_time - call_start > self.cfg[self.hubname]['hard_time_limit'] or \
number_of_turns > self.cfg[self.hubname]['hard_turn_limit']:
# hard hangup due to the hard limits
call_start = 0
number_of_turns = -1
vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))
if self.ncalls != 0 and not call_connected and s_last_dm_activity_time + 5.0 < current_time and ncalls >= self.ncalls:
break
# stop processes
vio_commands.send(Command('stop()', 'HUB', 'VoipIO'))
vad_commands.send(Command('stop()', 'HUB', 'VAD'))
asr_commands.send(Command('stop()', 'HUB', 'ASR'))
slu_commands.send(Command('stop()', 'HUB', 'SLU'))
dm_commands.send(Command('stop()', 'HUB', 'DM'))
nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
tts_commands.send(Command('stop()', 'HUB', 'TTS'))
# clean connections
for c in command_connections:
while c.poll():
c.recv()
for c in non_command_connections:
while c.poll():
c.recv()
# wait for processes to stop
# do not join, because in case of exception the join will not be successful
# vio.join()
# vad.join()
# asr.join()
# slu.join()
# dm.join()
# nlg.join()
# tts.join()
#cfg['Logging']['session_logger'].join()
except KeyboardInterrupt:
print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process(
).name
self.close_event.set()
return
except:
self.cfg['Logging']['system_logger'].exception(
'Uncaught exception in VHUB process.')
self.close_event.set()
raise
print 'Exiting: %s. Setting close event' % multiprocessing.current_process(
).name
self.close_event.set()
def run(self):
# AIO pipes
# used to send commands to VoipIO
aio_commands, aio_child_commands = multiprocessing.Pipe()
# I read from this connection recorded audio
aio_record, aio_child_record = multiprocessing.Pipe()
# I write in audio to be played
aio_play, aio_child_play = multiprocessing.Pipe()
# VAD pipes
# used to send commands to VAD
vad_commands, vad_child_commands = multiprocessing.Pipe()
# used to read output audio from VAD
vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()
# ASR pipes
# used to send commands to ASR
asr_commands, asr_child_commands = multiprocessing.Pipe()
# used to read ASR hypotheses
asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()
# SLU pipes
# used to send commands to SLU
slu_commands, slu_child_commands = multiprocessing.Pipe()
# used to read SLU hypotheses
slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()
# DM pipes
# used to send commands to DM
dm_commands, dm_child_commands = multiprocessing.Pipe()
# used to read DM actions
dm_actions_out, dm_child_actions = multiprocessing.Pipe()
# NLG pipes
# used to send commands to NLG
nlg_commands, nlg_child_commands = multiprocessing.Pipe()
# used to read NLG output
nlg_text_out, nlg_child_text = multiprocessing.Pipe()
# TTS pipes
# used to send commands to TTS
tts_commands, tts_child_commands = multiprocessing.Pipe()
command_connections = [
aio_commands, vad_commands, asr_commands, slu_commands,
dm_commands, nlg_commands, tts_commands
]
non_command_connections = [
aio_record, aio_child_record, aio_play, aio_child_play,
vad_audio_out, vad_child_audio_out, asr_hypotheses_out,
asr_child_hypotheses, slu_hypotheses_out, slu_child_hypotheses,
dm_actions_out, dm_child_actions, nlg_text_out, nlg_child_text
]
# create the hub components
close_event = multiprocessing.Event()
aio = WebIO(self.cfg, aio_child_commands, aio_child_record,
aio_child_play, close_event)
vad = VAD(self.cfg, vad_child_commands, aio_record,
vad_child_audio_out, close_event)
asr = ASR(self.cfg, asr_child_commands, vad_audio_out,
asr_child_hypotheses, close_event)
slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out,
slu_child_hypotheses, close_event)
dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out,
dm_child_actions, close_event)
nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out, nlg_child_text,
close_event)
tts = TTS(self.cfg, tts_child_commands, nlg_text_out, aio_play,
close_event)
# start the hub components
aio.start()
vad.start()
asr.start()
slu.start()
dm.start()
nlg.start()
tts.start()
# init the system
call_back_time = -1
call_back_uri = None
call_start = time.time()
self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
self.cfg['Logging']['system_logger'].session_system_log('config = ' +
str(self.cfg))
self.cfg['Logging']['session_logger'].session_start(
self.cfg['Logging']['system_logger'].get_session_dir_name())
self.cfg['Logging']['session_logger'].config('config = ' +
str(self.cfg))
self.cfg['Logging']['session_logger'].header(
self.cfg['Logging']["system_name"], self.cfg['Logging']["version"])
self.cfg['Logging']['session_logger'].input_source("aio")
while 1:
time.sleep(self.cfg['Hub']['main_loop_sleep_time'])
if call_back_time != -1 and call_back_time < time.time():
aio_commands.send(
Command('make_call(destination="%s")' % call_back_uri,
'HUB', 'AIO'))
call_back_time = -1
call_back_uri = None
if vad_commands.poll():
command = vad_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "speech_start":
pass
if command.parsed['__name__'] == "speech_end":
pass
if asr_commands.poll():
command = asr_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if slu_commands.poll():
command = slu_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if dm_commands.poll():
command = dm_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "hangup":
# prepare for ending the call
pass
if command.parsed['__name__'] == "dm_da_generated":
# record the time of the last system generated dialogue act
pass
if nlg_commands.poll():
command = nlg_commands.recv()
# TODO HACK
# self.cfg['Logging']['system_logger'].info(command)
if tts_commands.poll():
command = tts_commands.recv()
# TODO HACK
# self.cfg['Logging']['system_logger'].info(command)
current_time = time.time()
# stop processes
aio_commands.send(Command('stop()', 'HUB', 'AIO'))
vad_commands.send(Command('stop()', 'HUB', 'VAD'))
asr_commands.send(Command('stop()', 'HUB', 'ASR'))
slu_commands.send(Command('stop()', 'HUB', 'SLU'))
dm_commands.send(Command('stop()', 'HUB', 'DM'))
nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
tts_commands.send(Command('stop()', 'HUB', 'TTS'))
# clean connections
for c in command_connections:
while c.poll():
c.recv()
for c in non_command_connections:
while c.poll():
c.recv()
# wait for processes to stop
aio.join()
vad.join()
tts.join()
def run(self):
try:
# AIO pipes
# used to send commands to VoipIO
aio_commands, aio_child_commands = multiprocessing.Pipe()
# I read from this connection recorded audio
aio_record, aio_child_record = multiprocessing.Pipe()
# I write in audio to be played
aio_play, aio_child_play = multiprocessing.Pipe()
# VAD pipes
# used to send commands to VAD
vad_commands, vad_child_commands = multiprocessing.Pipe()
# used to read output audio from VAD
vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()
# ASR pipes
# used to send commands to ASR
asr_commands, asr_child_commands = multiprocessing.Pipe()
# used to read ASR hypotheses
asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()
# SLU pipes
# used to send commands to SLU
slu_commands, slu_child_commands = multiprocessing.Pipe()
# used to read SLU hypotheses
slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()
# DM pipes
# used to send commands to DM
dm_commands, dm_child_commands = multiprocessing.Pipe()
# used to read DM actions
dm_actions_out, dm_child_actions = multiprocessing.Pipe()
# NLG pipes
# used to send commands to NLG
nlg_commands, nlg_child_commands = multiprocessing.Pipe()
# used to read NLG output
nlg_text_out, nlg_child_text = multiprocessing.Pipe()
# TTS pipes
# used to send commands to TTS
tts_commands, tts_child_commands = multiprocessing.Pipe()
command_connections = [
aio_commands, vad_commands, asr_commands, slu_commands,
dm_commands, nlg_commands, tts_commands
]
non_command_connections = [
aio_record, aio_child_record, aio_play, aio_child_play,
vad_audio_out, vad_child_audio_out, asr_hypotheses_out,
asr_child_hypotheses, slu_hypotheses_out, slu_child_hypotheses,
dm_actions_out, dm_child_actions, nlg_text_out, nlg_child_text
]
close_event = multiprocessing.Event()
# create the hub components
aio = AudioIO(self.cfg, aio_child_commands, aio_child_record,
aio_child_play, close_event)
vad = VAD(self.cfg, vad_child_commands, aio_record,
vad_child_audio_out, close_event)
asr = ASR(self.cfg, asr_child_commands, vad_audio_out,
asr_child_hypotheses, close_event)
slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out,
slu_child_hypotheses, close_event)
dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out,
dm_child_actions, close_event)
nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out,
nlg_child_text, close_event)
tts = TTS(self.cfg, tts_child_commands, nlg_text_out, aio_play,
close_event)
# start the hub components
aio.start()
vad.start()
asr.start()
slu.start()
dm.start()
nlg.start()
tts.start()
self.cfg['Logging']['session_logger'].set_close_event(
self.close_event)
self.cfg['Logging']['session_logger'].set_cfg(self.cfg)
self.cfg['Logging']['session_logger'].start()
# init the system
call_start = 0
call_back_time = -1
call_back_uri = None
s_voice_activity = False
s_last_voice_activity_time = 0
u_voice_activity = False
u_last_voice_activity_time = 0
s_last_dm_activity_time = 0
hangup = False
call_start = time.time()
self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
self.cfg['Logging']['system_logger'].session_system_log(
'config = ' + str(self.cfg))
self.cfg['Logging']['session_logger'].session_start(
self.cfg['Logging']['system_logger'].get_session_dir_name())
self.cfg['Logging']['session_logger'].config('config = ' +
str(self.cfg))
self.cfg['Logging']['session_logger'].header(
self.cfg['Logging']["system_name"],
self.cfg['Logging']["version"])
self.cfg['Logging']['session_logger'].input_source("aio")
while 1:
# Check the close event.
if self.close_event.is_set():
print 'Received close event in: %s' % multiprocessing.current_process(
).name
return
time.sleep(self.cfg['Hub']['main_loop_sleep_time'])
if call_back_time != -1 and call_back_time < time.time():
aio_commands.send(Command('make_call(destination="%s")' % \
call_back_uri, 'HUB', 'AIO'))
call_back_time = -1
call_back_uri = None
if vad_commands.poll():
command = vad_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "speech_start":
u_voice_activity = True
if command.parsed['__name__'] == "speech_end":
u_voice_activity = False
u_last_voice_activity_time = time.time()
if asr_commands.poll():
command = asr_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if slu_commands.poll():
command = slu_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if dm_commands.poll():
command = dm_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if isinstance(command, Command):
if command.parsed['__name__'] == "hangup":
# prepare for ending the call
hangup = True
if command.parsed['__name__'] == "dm_da_generated":
# record the time of the last system generated dialogue act
s_last_dm_activity_time = time.time()
if nlg_commands.poll():
command = nlg_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
if tts_commands.poll():
command = tts_commands.recv()
self.cfg['Logging']['system_logger'].info(command)
current_time = time.time()
# stop processes
aio_commands.send(Command('stop()', 'HUB', 'AIO'))
vad_commands.send(Command('stop()', 'HUB', 'VAD'))
asr_commands.send(Command('stop()', 'HUB', 'ASR'))
slu_commands.send(Command('stop()', 'HUB', 'SLU'))
dm_commands.send(Command('stop()', 'HUB', 'DM'))
nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
tts_commands.send(Command('stop()', 'HUB', 'TTS'))
# clean connections
for c in command_connections:
while c.poll():
c.recv()
for c in non_command_connections:
while c.poll():
c.recv()
# wait for processes to stop
# do not join, because in case of exception the join will not be successful
# aio.join()
# vad.join()
# asr.join()
# slu.join()
# dm.join()
# nlg.join()
# tts.join()
#cfg['Logging']['session_logger'].join()
except KeyboardInterrupt:
print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process(
).name
self.close_event.set()
return
except:
self.cfg['Logging']['system_logger'].exception(
'Uncaught exception in AHUB process.')
self.close_event.set()
raise
print 'Exiting: %s. Setting close event' % multiprocessing.current_process(
).name
self.close_event.set()
def process_pending_commands(self):
"""Process all pending commands.
Available commands:
stop() - stop processing and exit the process
flush() - flush input buffers.
Now it only flushes the input connection.
It is not able flush data already send to the sound card.
call(dst) - make a call to the destination dst
transfer(dst) - transfer the existing call to the destination dst
hangup() - hang up the existing call
black_list(remote_uri, expire) - black list the specified uri until the expire time
- remote uri is get_user_from_uri provided by the on_call_confirmed call back
- expire is the time in second since the epoch that is time provided by time.time() function
Return True if the process should terminate.
"""
while self.local_commands:
command = self.local_commands.popleft()
if isinstance(command, Command):
if self.cfg['VoipIO']['debug']:
self.cfg['Logging']['system_logger'].debug(command)
if command.parsed['__name__'] == 'stop':
# discard all data in play buffer
while self.audio_play.poll():
data_play = self.audio_play.recv()
return True
if command.parsed['__name__'] == 'flush':
# discard all data in play buffer
while self.audio_play.poll():
data_play = self.audio_play.recv()
self.local_audio_play.clear()
self.mem_player.flush()
self.audio_playing = False
# flush the recorded data
while self.mem_capture.get_read_available():
data_rec = self.mem_capture.get_frame()
self.mem_capture.flush()
self.audio_recording = False
self.commands.send(Command("flushed()", 'VoipIO', 'HUB'))
return False
if command.parsed['__name__'] == 'flush_out':
# discard all data in play buffer
while self.audio_play.poll():
data_play = self.audio_play.recv()
self.local_audio_play.clear()
self.mem_player.flush()
self.audio_playing = False
self.commands.send(
Command("flushed_out()", 'VoipIO', 'HUB'))
return False
if command.parsed['__name__'] == 'make_call':
# make a call to the passed destination
self.make_call(command.parsed['destination'])
return False
if command.parsed['__name__'] == 'transfer':
# transfer the current call to the passed destination
self.transfer(command.parsed['destination'])
return False
if command.parsed['__name__'] == 'hangup':
# hangup the current call
self.hangup()
return False
if command.parsed['__name__'] == 'black_list':
# black list the passed remote uri, VoipIO will not accept any
# calls until the current time will be higher then the expire variable
remote_uri = command.parsed['remote_uri']
expire = int(command.parsed['expire'])
self.black_list[remote_uri] = expire
return False
raise VoipIOException('Unsupported command: ' + command)
return False
def read_write_audio(self, p, stream, wf, play_buffer):
"""Send some of the available data to the output.
It should be a non-blocking operation.
Therefore:
1) do not send more then play_buffer_frames
2) send only if stream.get_write_available() is more then the frame size
"""
if self.audio_play.poll():
while self.audio_play.poll() \
and len(play_buffer) < self.cfg['AudioIO']['play_buffer_size'] \
and stream.get_write_available() > self.cfg['Audio']['samples_per_frame']:
# send to play frames from input
data_play = self.audio_play.recv()
if isinstance(data_play, Frame):
stream.write(data_play.payload)
play_buffer.append(data_play)
if self.cfg['AudioIO']['debug']:
print '.',
sys.stdout.flush()
elif isinstance(data_play, Command):
if data_play.parsed['__name__'] == 'utterance_start':
self.commands.send(
Command('play_utterance_start()', 'AudioIO',
'HUB'))
if data_play.parsed['__name__'] == 'utterance_end':
self.commands.send(
Command('play_utterance_end()', 'AudioIO', 'HUB'))
else:
data_play = Frame(b"\x00\x00" *
self.cfg['Audio']['samples_per_frame'])
play_buffer.append(data_play)
if self.cfg['AudioIO']['debug']:
print '.',
sys.stdout.flush()
# record one packet of audio data
# it will be blocked until the data is recorded
data_rec = stream.read(self.cfg['Audio']['samples_per_frame'])
# send recorded data it must be read at the other end
self.audio_record.send(Frame(data_rec))
# get played audio block
data_play = play_buffer.pop(0)
# send played audio
# FIXME: I should save what I am playing
# self.audio_played.send(data_play)
# save the recorded and played data
data_stereo = bytearray()
for i in range(self.cfg['Audio']['samples_per_frame']):
data_stereo.extend(data_rec[i * 2])
data_stereo.extend(data_rec[i * 2 + 1])
# there might not be enough data to be played
# then add zeros
try:
data_stereo.extend(data_play[i * 2])
except IndexError:
data_stereo.extend(b'\x00')
try:
data_stereo.extend(data_play[i * 2 + 1])
except IndexError:
data_stereo.extend(b'\x00')
wf.writeframes(data_stereo)
# write one frame into the audio output
if wav:
data_play = wav.pop(0)
#print len(wav), len(data_play)
audio_play.send(Frame(data_play))
# read all VAD output audio
if vad_audio_out.poll():
data_vad = vad_audio_out.recv()
if isinstance(data_vad, Command):
if data_vad.parsed['__name__'] == 'speech_start':
print 'Speech start'
if data_vad.parsed['__name__'] == 'speech_end':
print 'Speech end'
# read all messages
for c in command_connections:
if c.poll():
command = c.recv()
print
print command
print
aio_commands.send(Command('stop()'))
vad_commands.send(Command('stop()'))
aio.join()
vad.join()
print
'config = ' + unicode(cfg))
cfg['Logging']['system_logger'].info(command)
cfg['Logging']['session_logger'].session_start(
cfg['Logging']['system_logger'].get_session_dir_name())
cfg['Logging']['session_logger'].config('config = ' +
unicode(cfg))
cfg['Logging']['session_logger'].header(
cfg['Logging']["system_name"],
cfg['Logging']["version"])
cfg['Logging']['session_logger'].input_source("voip")
elif command.parsed['__name__'] == "call_disconnected":
cfg['Logging']['system_logger'].info(command)
vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
cfg['Logging']['system_logger'].session_end()
cfg['Logging']['session_logger'].session_end()
# read all messages
for c in command_connections:
if c.poll():
command = c.recv()
system_logger.info(command)
# stop processes
vio_commands.send(Command('stop()', 'HUB', 'VoipIO'))
# clean connections
for c in command_connections:
def process_pending_commands(self):
"""Process all pending commands.
Available commands:
stop() - stop processing and exit the process
flush() - flush input buffers.
Now it only flushes the input connection.
Return True if the process should terminate.
"""
while self.commands.poll():
command = self.commands.recv()
if self.cfg['DM']['debug']:
self.cfg['Logging']['system_logger'].debug(command)
if isinstance(command, Command):
if command.parsed['__name__'] == 'stop':
return True
if command.parsed['__name__'] == 'flush':
# discard all data in in input buffers
while self.slu_hypotheses_in.poll():
data_in = self.slu_hypotheses_in.recv()
self.dm.end_dialogue()
self.commands.send(Command("flushed()", 'DM', 'HUB'))
return False
if command.parsed['__name__'] == 'new_dialogue':
self.epilogue_state = None
self.dm.new_dialogue()
self.cfg['Logging']['session_logger'].turn("system")
self.dm.log_state()
# I should generate the first DM output
da = self.dm.da_out()
if self.cfg['DM']['debug']:
s = []
s.append("DM Output")
s.append("-"*60)
s.append(unicode(da))
s.append("")
s = '\n'.join(s)
self.cfg['Logging']['system_logger'].debug(s)
self.cfg['Logging']['session_logger'].dialogue_act("system", da)
self.commands.send(DMDA(da, 'DM', 'HUB'))
return False
if command.parsed['__name__'] == 'end_dialogue':
self.dm.end_dialogue()
return False
if command.parsed['__name__'] == 'timeout':
# check whether there is a looong silence
# if yes then inform the DM
silence_time = command.parsed['silence_time']
cn = DialogueActConfusionNetwork()
cn.add(1.0, DialogueActItem('silence','time', silence_time))
# process the input DA
self.dm.da_in(cn)
self.cfg['Logging']['session_logger'].turn("system")
self.dm.log_state()
if self.epilogue_state and float(silence_time) > 5.0:
# a user was silent for too long, therefore hung up
self.cfg['Logging']['session_logger'].dialogue_act("system", self.epilogue_da)
self.commands.send(DMDA(self.epilogue_da, 'DM', 'HUB'))
self.commands.send(Command('hangup()', 'DM', 'HUB'))
else:
da = self.dm.da_out()
if self.cfg['DM']['debug']:
s = []
s.append("DM Output")
s.append("-"*60)
s.append(unicode(da))
s.append("")
s = '\n'.join(s)
self.cfg['Logging']['system_logger'].debug(s)
self.cfg['Logging']['session_logger'].dialogue_act("system", da)
self.commands.send(DMDA(da, 'DM', 'HUB'))
if da.has_dat("bye"):
self.commands.send(Command('hangup()', 'DM', 'HUB'))
return False
return False
def read_audio_write_asr_hypotheses(self):
# Read input audio.
if self.local_audio_in:
if len(self.local_audio_in) > 40:
print "ASR unprocessed frames:", len(self.local_audio_in)
if len(self.local_audio_in) > 200:
print "ASR too many unprocessed frames:", len(
self.local_audio_in)
print " skipping everything until the end of the segment:", len(
self.local_audio_in)
while len(self.local_audio_in) > 2 and isinstance(
self.local_audio_in[0], Frame):
skip = self.local_audio_in.popleft()
# read recorded audio
data_rec = self.local_audio_in.popleft()
if isinstance(data_rec, Frame):
if self.recognition_on:
self.asr.rec_in(data_rec)
elif isinstance(data_rec, Command):
dr_speech_start = False
fname = None
if data_rec.parsed['__name__'] == "speech_start":
# check whether there are more then one speech segments
segments = [
cmd for cmd in self.local_audio_in
if isinstance(cmd, Command)
and cmd.parsed['__name__'] == "speech_start"
]
if len(segments):
# there are multiple unprocessed segments in the queue
# remove all unprocessed segments except the last
print "ASR too many unprocessed speech segments:", len(
segments)
print " removed all segments but the last"
removed_segments = 0
while removed_segments < len(segments):
data_rec = self.local_audio_in.popleft()
if isinstance(data_rec,
Command) and data_rec.parsed[
'__name__'] == "speech_start":
removed_segments += 1
dr_speech_start = "speech_start"
fname = data_rec.parsed['fname']
elif data_rec.parsed['__name__'] == "speech_end":
dr_speech_start = "speech_end"
fname = data_rec.parsed['fname']
# Check consistency of the input command.
if dr_speech_start:
if ((not self.recognition_on
and dr_speech_start != "speech_start")
or (self.recognition_on
and dr_speech_start != "speech_end")):
msg = ('Commands received by the ASR component are '
'inconsistent (recognition_on: {rec}; the new '
'command: {cmd}').format(
rec=self.recognition_on,
cmd=dr_speech_start)
self.system_logger.exception(msg)
if dr_speech_start == "speech_start":
self.commands.send(
Command('asr_start(fname="%s")' % fname, 'ASR', 'HUB'))
self.recognition_on = True
if self.cfg['ASR']['debug']:
self.system_logger.debug(
'ASR: speech_start(fname="%s")' % fname)
elif dr_speech_start == "speech_end":
self.recognition_on = False
if self.cfg['ASR']['debug']:
self.system_logger.debug(
'ASR: speech_end(fname="%s")' % fname)
try:
asr_hyp = self.asr.hyp_out()
if self.cfg['ASR']['debug']:
msg = list()
msg.append("ASR Hypothesis")
msg.append("-" * 60)
msg.append(unicode(asr_hyp))
msg.append(u"")
msg = u'\n'.join(msg)
self.system_logger.debug(msg)
except (ASRException, JuliusASRTimeoutException):
self.system_logger.debug("Julius ASR Result Timeout.")
if self.cfg['ASR']['debug']:
msg = list()
msg.append("ASR Alternative hypothesis")
msg.append("-" * 60)
msg.append("sil")
msg.append("")
msg = u'\n'.join(msg)
self.system_logger.debug(msg)
asr_hyp = UtteranceConfusionNetwork()
asr_hyp.add([
[1.0, "_other_"],
])
# The ASR component can return either NBList or a confusion
# network.
if isinstance(asr_hyp, UtteranceNBList):
self.session_logger.asr("user", fname, asr_hyp, None)
elif isinstance(asr_hyp, UtteranceConfusionNetwork):
self.session_logger.asr("user", fname,
asr_hyp.get_utterance_nblist(),
asr_hyp)
else:
self.session_logger.asr("user", fname, [(-1, asr_hyp)],
None)
self.commands.send(
Command('asr_end(fname="%s")' % fname, 'ASR', 'HUB'))
self.commands.send(ASRHyp(asr_hyp, fname=fname))
self.asr_hypotheses_out.send(ASRHyp(asr_hyp, fname=fname))
else:
raise ASRException('Unsupported input.')