tts_text_in, tts_child_text_in = multiprocessing.Pipe()
    # used to send TTS text

    command_connections = (vio_commands, vad_commands, asr_commands, tts_commands)

    non_command_connections = (vio_record, vio_child_record,
                               vio_play, vio_child_play,
                               vad_audio_out, vad_child_audio_out,
                               asr_hypotheses_out, asr_child_hypotheses,
                               tts_text_in, tts_child_text_in)

    close_event = multiprocessing.Event()

    vio = VoipIO(cfg, vio_child_commands, vio_child_record, vio_child_play, close_event)
    vad = VAD(cfg, vad_child_commands, vio_record, vad_child_audio_out, close_event)
    asr = ASR(cfg, asr_child_commands, vad_audio_out, asr_child_hypotheses, close_event)
    tts = TTS(cfg, tts_child_commands, tts_child_text_in, vio_play, close_event)

    vio.start()
    vad.start()
    asr.start()
    tts.start()

    # Actively call a number configured.
    #vio_commands.send(Command('make_call(destination="\'Bejcek Eduard\' <sip:4366@SECRET:5066>")', 'HUB', 'VoipIO'))
    #vio_commands.send(Command('make_call(destination="sip:4366@SECRET:5066")', 'HUB', 'VoipIO'))
    #vio_commands.send(Command('make_call(destination="4366")', 'HUB', 'VoipIO'))
    #vio_commands.send(Command('make_call(destination="155")', 'HUB', 'VoipIO'))

    count = 0
Example #2
0
    wav = audio.load_wav(cfg, './resources/test16k-mono.wav')
    # split audio into frames
    wav = various.split_to_bins(wav, 2 * cfg['Audio']['samples_per_frame'])
    # remove the last frame

    aio_commands, aio_child_commands = multiprocessing.Pipe()  # used to send commands to AudioIO
    audio_record, child_audio_record = multiprocessing.Pipe()  # I read from this connection recorded audio
    audio_play, child_audio_play = multiprocessing.Pipe( )     # I write in audio to be played

    vad_commands, vad_child_commands = multiprocessing.Pipe()  # used to send commands to VAD
    vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()# used to read output audio from VAD

    close_event = multiprocessing.Event()

    aio = AudioIO(cfg, aio_child_commands, child_audio_record, child_audio_play, close_event)
    vad = VAD(cfg, vad_child_commands, audio_record, vad_child_audio_out, close_event)

    command_connections = [aio_commands, vad_commands]

    aio.start()
    vad.start()

    count = 0
    max_count = 5000
    while count < max_count:
        time.sleep(cfg['Hub']['main_loop_sleep_time'])
        count += 1

        # write one frame into the audio output
        if wav:
            data_play = wav.pop(0)
Example #3
0
    def run(self):
        try:
            cfg = self.cfg

            vio_commands, vio_child_commands = multiprocessing.Pipe(
            )  # used to send commands to VoipIO
            vio_record, vio_child_record = multiprocessing.Pipe(
            )  # I read from this connection recorded audio
            vio_play, vio_child_play = multiprocessing.Pipe(
            )  # I write in audio to be played

            vad_commands, vad_child_commands = multiprocessing.Pipe(
            )  # used to send commands to VAD
            vad_audio_out, vad_child_audio_out = multiprocessing.Pipe(
            )  # used to read output audio from VAD

            asr_commands, asr_child_commands = multiprocessing.Pipe(
            )  # used to send commands to ASR
            asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe(
            )  # used to read ASR hypotheses

            slu_commands, slu_child_commands = multiprocessing.Pipe(
            )  # used to send commands to SLU
            slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe(
            )  # used to read SLU hypotheses

            dm_commands, dm_child_commands = multiprocessing.Pipe(
            )  # used to send commands to DM
            dm_actions_out, dm_child_actions = multiprocessing.Pipe(
            )  # used to read DM actions

            nlg_commands, nlg_child_commands = multiprocessing.Pipe(
            )  # used to send commands to NLG
            nlg_text_out, nlg_child_text = multiprocessing.Pipe(
            )  # used to read NLG output

            tts_commands, tts_child_commands = multiprocessing.Pipe(
            )  # used to send commands to TTS

            command_connections = [
                vio_commands, vad_commands, asr_commands, slu_commands,
                dm_commands, nlg_commands, tts_commands
            ]

            non_command_connections = [
                vio_record, vio_child_record, vio_play, vio_child_play,
                vad_audio_out, vad_child_audio_out, asr_hypotheses_out,
                asr_child_hypotheses, slu_hypotheses_out, slu_child_hypotheses,
                dm_actions_out, dm_child_actions, nlg_text_out, nlg_child_text
            ]

            vio = self.voice_io_cls(self.cfg, vio_child_commands,
                                    vio_child_record, vio_child_play,
                                    self.close_event)
            vad = VAD(self.cfg, vad_child_commands, vio_record,
                      vad_child_audio_out, self.close_event)
            asr = ASR(self.cfg, asr_child_commands, vad_audio_out,
                      asr_child_hypotheses, self.close_event)
            slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out,
                      slu_child_hypotheses, self.close_event)
            dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out,
                    dm_child_actions, self.close_event)
            nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out,
                      nlg_child_text, self.close_event)
            tts = TTS(self.cfg, tts_child_commands, nlg_text_out, vio_play,
                      self.close_event)

            vio.start()
            vad.start()
            asr.start()
            slu.start()
            dm.start()
            nlg.start()
            tts.start()

            self.write_pid_file([['vio', vio.pid], ['vad', vad.pid],
                                 ['asr', asr.pid], ['slu', slu.pid],
                                 ['dm', dm.pid], ['nlg', nlg.pid],
                                 ['tts', tts.pid]])

            cfg['Logging']['session_logger'].set_close_event(self.close_event)
            cfg['Logging']['session_logger'].set_cfg(cfg)
            cfg['Logging']['session_logger'].start()
            cfg['Logging']['session_logger'].cancel_join_thread()

            # init the system
            call_start = 0
            call_back_time = -1
            call_back_uri = None
            number_of_turns = -1

            s_voice_activity = False
            s_last_voice_activity_time = 0
            u_voice_activity = False
            u_last_voice_activity_time = 0

            s_last_dm_activity_time = 0

            u_last_input_timeout = 0

            call_connected = False
            hangup = False
            ncalls = 0

            outstanding_nlg_da = None

            call_db = CallDB(self.cfg, self.cfg[self.hubname]['call_db'],
                             self.cfg[self.hubname]['period'])
            #call_db.log()

            while 1:
                # Check the close event.
                if self.close_event.is_set():
                    print 'Received close event in: %s' % multiprocessing.current_process(
                    ).name
                    return

                time.sleep(self.cfg['Hub']['main_loop_sleep_time'])

                if call_back_time != -1 and call_back_time < time.time():
                    vio_commands.send(
                        Command('make_call(destination="%s")' % call_back_uri,
                                'HUB', 'VoipIO'))
                    call_back_time = -1
                    call_back_uri = None

                # read all messages
                if vio_commands.poll():
                    command = vio_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):

                        if command.parsed['__name__'] == "incoming_call":
                            self.cfg['Logging']['system_logger'].session_start(
                                command.parsed['remote_uri'])
                            self.cfg['Logging'][
                                'session_logger'].session_start(
                                    self.cfg['Logging']
                                    ['system_logger'].get_session_dir_name())

                            self.cfg['Logging'][
                                'system_logger'].session_system_log(
                                    'config = ' + unicode(self.cfg))
                            self.cfg['Logging']['session_logger'].config(
                                'config = ' + unicode(self.cfg))
                            self.cfg['Logging']['session_logger'].header(
                                self.cfg['Logging']["system_name"],
                                self.cfg['Logging']["version"])
                            self.cfg['Logging']['session_logger'].input_source(
                                "voip")

                            self.cfg['Analytics'].start_session(
                                command.parsed['remote_uri'])
                            self.cfg['Analytics'].track_event(
                                'vhub', 'incoming_call',
                                command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "rejected_call":
                            call_back_time = time.time() + self.cfg['VoipHub'][
                                'wait_time_before_calling_back']
                            call_back_uri = command.parsed['remote_uri']

                            self.cfg['Analytics'].track_event(
                                'vhub', 'rejected_call',
                                command.parsed['remote_uri'])

                        if command.parsed[
                                '__name__'] == "rejected_call_from_blacklisted_uri":
                            remote_uri = command.parsed['remote_uri']

                            num_all_calls, total_time, last_period_num_calls, last_period_total_time, last_period_num_short_calls = call_db.get_uri_stats(
                                remote_uri)

                            m = []
                            m.append('')
                            m.append('=' * 120)
                            m.append(
                                'Rejected incoming call from blacklisted URI: %s'
                                % remote_uri)
                            m.append('-' * 120)
                            m.append('Total calls:                  %d' %
                                     num_all_calls)
                            m.append('Total time (min):             %0.1f' %
                                     (total_time / 60.0, ))
                            m.append('Last period short calls:      %d' %
                                     last_period_num_short_calls)
                            m.append('Last period total calls:      %d' %
                                     last_period_num_calls)
                            m.append('Last period total time (min): %0.1f' %
                                     (last_period_total_time / 60.0, ))
                            m.append('=' * 120)
                            m.append('')
                            self.cfg['Logging']['system_logger'].info(
                                '\n'.join(m))

                            self.cfg['Analytics'].track_event(
                                'vhub', 'rejected_call_from_blacklisted_uri',
                                command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "call_connecting":
                            self.cfg['Analytics'].track_event(
                                'vhub', 'call_connecting',
                                command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "call_confirmed":
                            remote_uri = command.parsed['remote_uri']
                            num_all_calls, total_time, last_period_num_calls, last_period_total_time, last_period_num_short_calls = call_db.get_uri_stats(
                                remote_uri)

                            m = []
                            m.append('')
                            m.append('=' * 120)
                            m.append('Incoming call from :          %s' %
                                     remote_uri)
                            m.append('-' * 120)
                            m.append('Total calls:                  %d' %
                                     num_all_calls)
                            m.append('Total time (min):             %0.1f' %
                                     (total_time / 60.0, ))
                            m.append('Last period short calls:      %d' %
                                     last_period_num_short_calls)
                            m.append('Last period total calls:      %d' %
                                     last_period_num_calls)
                            m.append('Last period total time (min): %0.1f' %
                                     (last_period_total_time / 60.0, ))
                            m.append('-' * 120)

                            if last_period_num_calls > self.cfg[self.hubname]['last_period_max_num_calls'] or \
                                    last_period_total_time > self.cfg[self.hubname]['last_period_max_total_time'] or \
                                    last_period_num_short_calls > self.cfg[self.hubname]['last_period_max_num_short_calls'] :
                                # prepare for ending the call
                                call_connected = True

                                call_start = time.time()
                                number_of_turns = -1
                                s_voice_activity = True
                                s_last_voice_activity_time = time.time()
                                u_voice_activity = False
                                u_last_voice_activity_time = time.time()
                                u_last_input_timeout = time.time()
                                hangup = True

                                self.cfg['Logging']['session_logger'].turn(
                                    "system")
                                tts_commands.send(
                                    Command(
                                        'synthesize(text="%s",log="true")' %
                                        self.cfg[self.hubname]
                                        ['limit_reached_message'], 'HUB',
                                        'TTS'))
                                vio_commands.send(
                                    Command(
                                        'black_list(remote_uri="%s",expire="%d")'
                                        % (remote_uri, time.time() + self.cfg[
                                            self.hubname]['blacklist_for']),
                                        'HUB', 'VoipIO'))
                                m.append('CALL REJECTED')
                            else:
                                # init the system
                                call_connected = True

                                call_start = time.time()
                                number_of_turns = 0

                                s_voice_activity = False
                                s_last_voice_activity_time = 0
                                u_voice_activity = False
                                u_last_voice_activity_time = time.time()
                                u_last_input_timeout = time.time()
                                hangup = False

                                dm_commands.send(
                                    Command('new_dialogue()', 'HUB', 'DM'))
                                m.append('CALL ACCEPTED')

                            m.append('=' * 120)
                            m.append('')
                            self.cfg['Logging']['system_logger'].info(
                                '\n'.join(m))

                            call_db.track_confirmed_call(remote_uri)
                            self.cfg['Analytics'].track_event(
                                'vhub', 'call_confirmed',
                                command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "call_disconnected":
                            # flush vio, when flushed, vad will be flushed
                            vio_commands.send(
                                Command('flush()', 'HUB', 'VoipIO'))

                            self.cfg['Logging']['system_logger'].session_end()
                            self.cfg['Logging']['session_logger'].session_end()

                            remote_uri = command.parsed['remote_uri']
                            call_db.track_disconnected_call(remote_uri)

                            call_connected = False
                            number_of_turns = -1
                            ncalls += 1

                            self.cfg['Analytics'].track_event(
                                'vhub', 'call_disconnected',
                                command.parsed['remote_uri'])

                            dm_commands.send(
                                Command('prepare_new_dialogue()', 'HUB', 'DM'))

                        if command.parsed[
                                '__name__'] == "play_utterance_start":
                            s_voice_activity = True
                            s_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "play_utterance_end":
                            s_voice_activity = False
                            s_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "flushed":
                            # flush vad, when flushed, asr will be flushed
                            vad_commands.send(Command('flush()', 'HUB', 'VAD'))

                        if command.parsed['__name__'] == "flushed_out":
                            # process the outstanding DA if necessary
                            if outstanding_nlg_da:
                                nlg_commands.send(
                                    DMDA(outstanding_nlg_da, 'HUB', 'NLG'))
                                outstanding_nlg_da = None

                if vad_commands.poll():
                    command = vad_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "speech_start":
                            u_voice_activity = True

                            # interrupt the talking system
                            # this will be replaced with pausing the system when the necessary extension of pjsip
                            # is implemented
                            if s_voice_activity and s_last_voice_activity_time + 0.02 < current_time:
                                # if the system is still talking then flush the output
                                self.cfg['Logging']['session_logger'].barge_in(
                                    "system")

                                # when a user barge in into the output, all the output pipe line
                                # must be flushed
                                nlg_commands.send(
                                    Command('flush()', 'HUB', 'NLG'))
                                s_voice_activity = False
                                s_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "speech_end":
                            u_voice_activity = False
                            u_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "flushed":
                            # flush asr, when flushed, slu will be flushed
                            asr_commands.send(Command('flush()', 'HUB', 'ASR'))

                if asr_commands.poll():
                    command = asr_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, ASRHyp):
                        vio_commands.send(command)
                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush slu, when flushed, dm will be flushed
                            slu_commands.send(Command('flush()', 'HUB', 'SLU'))

                if slu_commands.poll():
                    command = slu_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush dm, when flushed, nlg will be flushed
                            dm_commands.send(Command('flush()', 'HUB', 'DM'))
                            dm_commands.send(
                                Command('end_dialogue()', 'HUB', 'DM'))

                if dm_commands.poll():
                    command = dm_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "hangup":
                            # prepare for ending the call
                            hangup = True
                            self.cfg['Analytics'].track_event(
                                'vhub', 'system_hangup')

                        if command.parsed['__name__'] == "flushed":
                            # flush nlg, when flushed, tts will be flushed
                            nlg_commands.send(Command('flush()', 'HUB', 'NLG'))

                    elif isinstance(command, DMDA):
                        # record the time of the last system generated dialogue act
                        s_last_dm_activity_time = time.time()
                        number_of_turns += 1

                        if command.da != "silence()":
                            # if the DM generated non-silence dialogue act, then continue in processing it

                            if s_voice_activity and s_last_voice_activity_time + 0.02 < current_time:
                                # if the system is still talking then flush the output
                                self.cfg['Logging']['session_logger'].barge_in(
                                    "system")

                                # when a user barge in into the output, all the output pipe line
                                # must be flushed
                                nlg_commands.send(
                                    Command('flush()', 'HUB', 'NLG'))
                                s_voice_activity = False
                                s_last_voice_activity_time = time.time()

                                # the DA will be send when all the following components are flushed
                                outstanding_nlg_da = command.da

                            else:
                                nlg_commands.send(
                                    DMDA(command.da, "HUB", "NLG"))

                if nlg_commands.poll():
                    command = nlg_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, TTSText):
                        vio_commands.send(command)
                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush tts, when flushed, vio will be flushed
                            tts_commands.send(Command('flush()', 'HUB', 'TTS'))

                if tts_commands.poll():
                    command = tts_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush vio_out
                            vio_commands.send(
                                Command('flush_out()', 'HUB', 'VIO'))

                current_time = time.time()

                s_diff = current_time - s_last_voice_activity_time
                u_diff = current_time - u_last_voice_activity_time
                t_diff = current_time - u_last_input_timeout
                timeout = self.cfg['DM']['input_timeout']

                if call_connected and \
                    not s_voice_activity and not u_voice_activity and \
                    s_diff > timeout and \
                    u_diff > timeout and \
                    t_diff > timeout:

                    u_last_input_timeout = time.time()
                    print 'timeout!!!!!', time.time()
                    dm_commands.send(
                        Command(
                            'timeout(silence_time="%0.3f")' %
                            min(s_diff, u_diff), 'HUB', 'DM'))

                if hangup and s_last_dm_activity_time + 2.0 < current_time and \
                    s_voice_activity == False and s_last_voice_activity_time + 2.0 < current_time:
                    # we are ready to hangup only when all voice activity is finished
                    hangup = False
                    vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))

                if number_of_turns != -1 and current_time - call_start > self.cfg[self.hubname]['hard_time_limit'] or \
                    number_of_turns > self.cfg[self.hubname]['hard_turn_limit']:
                    # hard hangup due to the hard limits
                    call_start = 0
                    number_of_turns = -1
                    vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))

                if self.ncalls != 0 and not call_connected and s_last_dm_activity_time + 5.0 < current_time and ncalls >= self.ncalls:
                    break

            # stop processes
            vio_commands.send(Command('stop()', 'HUB', 'VoipIO'))
            vad_commands.send(Command('stop()', 'HUB', 'VAD'))
            asr_commands.send(Command('stop()', 'HUB', 'ASR'))
            slu_commands.send(Command('stop()', 'HUB', 'SLU'))
            dm_commands.send(Command('stop()', 'HUB', 'DM'))
            nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
            tts_commands.send(Command('stop()', 'HUB', 'TTS'))

            # clean connections
            for c in command_connections:
                while c.poll():
                    c.recv()

            for c in non_command_connections:
                while c.poll():
                    c.recv()

            # wait for processes to stop
            # do not join, because in case of exception the join will not be successful
            # vio.join()
            # vad.join()
            # asr.join()
            # slu.join()
            # dm.join()
            # nlg.join()
            # tts.join()
            #cfg['Logging']['session_logger'].join()

        except KeyboardInterrupt:
            print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process(
            ).name
            self.close_event.set()
            return
        except:
            self.cfg['Logging']['system_logger'].exception(
                'Uncaught exception in VHUB process.')
            self.close_event.set()
            raise

        print 'Exiting: %s. Setting close event' % multiprocessing.current_process(
        ).name
        self.close_event.set()
Example #4
0
    def run(self):
        # AIO pipes
        # used to send commands to VoipIO
        aio_commands, aio_child_commands = multiprocessing.Pipe()
        # I read from this connection recorded audio
        aio_record, aio_child_record = multiprocessing.Pipe()
        # I write in audio to be played
        aio_play, aio_child_play = multiprocessing.Pipe()

        # VAD pipes
        # used to send commands to VAD
        vad_commands, vad_child_commands = multiprocessing.Pipe()
        # used to read output audio from VAD
        vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()

        # ASR pipes
        # used to send commands to ASR
        asr_commands, asr_child_commands = multiprocessing.Pipe()
        # used to read ASR hypotheses
        asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()

        # SLU pipes
        # used to send commands to SLU
        slu_commands, slu_child_commands = multiprocessing.Pipe()
        # used to read SLU hypotheses
        slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()

        # DM pipes
        # used to send commands to DM
        dm_commands, dm_child_commands = multiprocessing.Pipe()
        # used to read DM actions
        dm_actions_out, dm_child_actions = multiprocessing.Pipe()

        # NLG pipes
        # used to send commands to NLG
        nlg_commands, nlg_child_commands = multiprocessing.Pipe()
        # used to read NLG output
        nlg_text_out, nlg_child_text = multiprocessing.Pipe()

        # TTS pipes
        # used to send commands to TTS
        tts_commands, tts_child_commands = multiprocessing.Pipe()

        command_connections = [
            aio_commands, vad_commands, asr_commands, slu_commands,
            dm_commands, nlg_commands, tts_commands
        ]

        non_command_connections = [
            aio_record, aio_child_record, aio_play, aio_child_play,
            vad_audio_out, vad_child_audio_out, asr_hypotheses_out,
            asr_child_hypotheses, slu_hypotheses_out, slu_child_hypotheses,
            dm_actions_out, dm_child_actions, nlg_text_out, nlg_child_text
        ]

        # create the hub components
        close_event = multiprocessing.Event()
        aio = WebIO(self.cfg, aio_child_commands, aio_child_record,
                    aio_child_play, close_event)
        vad = VAD(self.cfg, vad_child_commands, aio_record,
                  vad_child_audio_out, close_event)
        asr = ASR(self.cfg, asr_child_commands, vad_audio_out,
                  asr_child_hypotheses, close_event)
        slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out,
                  slu_child_hypotheses, close_event)
        dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out,
                dm_child_actions, close_event)
        nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out, nlg_child_text,
                  close_event)
        tts = TTS(self.cfg, tts_child_commands, nlg_text_out, aio_play,
                  close_event)

        # start the hub components
        aio.start()
        vad.start()
        asr.start()
        slu.start()
        dm.start()
        nlg.start()
        tts.start()

        # init the system
        call_back_time = -1
        call_back_uri = None
        call_start = time.time()

        self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
        self.cfg['Logging']['system_logger'].session_system_log('config = ' +
                                                                str(self.cfg))

        self.cfg['Logging']['session_logger'].session_start(
            self.cfg['Logging']['system_logger'].get_session_dir_name())
        self.cfg['Logging']['session_logger'].config('config = ' +
                                                     str(self.cfg))
        self.cfg['Logging']['session_logger'].header(
            self.cfg['Logging']["system_name"], self.cfg['Logging']["version"])
        self.cfg['Logging']['session_logger'].input_source("aio")

        while 1:
            time.sleep(self.cfg['Hub']['main_loop_sleep_time'])

            if call_back_time != -1 and call_back_time < time.time():
                aio_commands.send(
                    Command('make_call(destination="%s")' % call_back_uri,
                            'HUB', 'AIO'))
                call_back_time = -1
                call_back_uri = None

            if vad_commands.poll():
                command = vad_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "speech_start":
                        pass
                    if command.parsed['__name__'] == "speech_end":
                        pass

            if asr_commands.poll():
                command = asr_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

            if slu_commands.poll():
                command = slu_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

            if dm_commands.poll():
                command = dm_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "hangup":
                        # prepare for ending the call
                        pass

                    if command.parsed['__name__'] == "dm_da_generated":
                        # record the time of the last system generated dialogue act
                        pass

            if nlg_commands.poll():
                command = nlg_commands.recv()
                # TODO HACK
                # self.cfg['Logging']['system_logger'].info(command)

            if tts_commands.poll():
                command = tts_commands.recv()
                # TODO HACK
                # self.cfg['Logging']['system_logger'].info(command)

            current_time = time.time()

        # stop processes
        aio_commands.send(Command('stop()', 'HUB', 'AIO'))
        vad_commands.send(Command('stop()', 'HUB', 'VAD'))
        asr_commands.send(Command('stop()', 'HUB', 'ASR'))
        slu_commands.send(Command('stop()', 'HUB', 'SLU'))
        dm_commands.send(Command('stop()', 'HUB', 'DM'))
        nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
        tts_commands.send(Command('stop()', 'HUB', 'TTS'))

        # clean connections
        for c in command_connections:
            while c.poll():
                c.recv()

        for c in non_command_connections:
            while c.poll():
                c.recv()

        # wait for processes to stop
        aio.join()
        vad.join()
        tts.join()
Example #5
0
File: ahub.py Project: tkraut/alex
    def run(self):
        try:
            # AIO pipes
            # used to send commands to VoipIO
            aio_commands, aio_child_commands = multiprocessing.Pipe()
            # I read from this connection recorded audio
            aio_record, aio_child_record = multiprocessing.Pipe()
            # I write in audio to be played
            aio_play, aio_child_play = multiprocessing.Pipe()

            # VAD pipes
            # used to send commands to VAD
            vad_commands, vad_child_commands = multiprocessing.Pipe()
            # used to read output audio from VAD
            vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()

            # ASR pipes
            # used to send commands to ASR
            asr_commands, asr_child_commands = multiprocessing.Pipe()
            # used to read ASR hypotheses
            asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()

            # SLU pipes
            # used to send commands to SLU
            slu_commands, slu_child_commands = multiprocessing.Pipe()
            # used to read SLU hypotheses
            slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()

            # DM pipes
            # used to send commands to DM
            dm_commands, dm_child_commands = multiprocessing.Pipe()
            # used to read DM actions
            dm_actions_out, dm_child_actions = multiprocessing.Pipe()

            # NLG pipes
            # used to send commands to NLG
            nlg_commands, nlg_child_commands = multiprocessing.Pipe()
            # used to read NLG output
            nlg_text_out, nlg_child_text = multiprocessing.Pipe()

            # TTS pipes
            # used to send commands to TTS
            tts_commands, tts_child_commands = multiprocessing.Pipe()

            command_connections = [
                aio_commands, vad_commands, asr_commands, slu_commands,
                dm_commands, nlg_commands, tts_commands
            ]

            non_command_connections = [
                aio_record, aio_child_record, aio_play, aio_child_play,
                vad_audio_out, vad_child_audio_out, asr_hypotheses_out,
                asr_child_hypotheses, slu_hypotheses_out, slu_child_hypotheses,
                dm_actions_out, dm_child_actions, nlg_text_out, nlg_child_text
            ]

            close_event = multiprocessing.Event()

            # create the hub components
            aio = AudioIO(self.cfg, aio_child_commands, aio_child_record,
                          aio_child_play, close_event)
            vad = VAD(self.cfg, vad_child_commands, aio_record,
                      vad_child_audio_out, close_event)
            asr = ASR(self.cfg, asr_child_commands, vad_audio_out,
                      asr_child_hypotheses, close_event)
            slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out,
                      slu_child_hypotheses, close_event)
            dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out,
                    dm_child_actions, close_event)
            nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out,
                      nlg_child_text, close_event)
            tts = TTS(self.cfg, tts_child_commands, nlg_text_out, aio_play,
                      close_event)

            # start the hub components
            aio.start()
            vad.start()
            asr.start()
            slu.start()
            dm.start()
            nlg.start()
            tts.start()

            self.cfg['Logging']['session_logger'].set_close_event(
                self.close_event)
            self.cfg['Logging']['session_logger'].set_cfg(self.cfg)
            self.cfg['Logging']['session_logger'].start()

            # init the system
            call_start = 0
            call_back_time = -1
            call_back_uri = None

            s_voice_activity = False
            s_last_voice_activity_time = 0
            u_voice_activity = False
            u_last_voice_activity_time = 0

            s_last_dm_activity_time = 0

            hangup = False

            call_start = time.time()

            self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
            self.cfg['Logging']['system_logger'].session_system_log(
                'config = ' + str(self.cfg))

            self.cfg['Logging']['session_logger'].session_start(
                self.cfg['Logging']['system_logger'].get_session_dir_name())
            self.cfg['Logging']['session_logger'].config('config = ' +
                                                         str(self.cfg))
            self.cfg['Logging']['session_logger'].header(
                self.cfg['Logging']["system_name"],
                self.cfg['Logging']["version"])
            self.cfg['Logging']['session_logger'].input_source("aio")

            while 1:
                # Check the close event.
                if self.close_event.is_set():
                    print 'Received close event in: %s' % multiprocessing.current_process(
                    ).name
                    return

                time.sleep(self.cfg['Hub']['main_loop_sleep_time'])

                if call_back_time != -1 and call_back_time < time.time():
                    aio_commands.send(Command('make_call(destination="%s")' % \
                                              call_back_uri, 'HUB', 'AIO'))
                    call_back_time = -1
                    call_back_uri = None

                if vad_commands.poll():
                    command = vad_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "speech_start":
                            u_voice_activity = True
                        if command.parsed['__name__'] == "speech_end":
                            u_voice_activity = False
                            u_last_voice_activity_time = time.time()

                if asr_commands.poll():
                    command = asr_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                if slu_commands.poll():
                    command = slu_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                if dm_commands.poll():
                    command = dm_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "hangup":
                            # prepare for ending the call
                            hangup = True

                        if command.parsed['__name__'] == "dm_da_generated":
                            # record the time of the last system generated dialogue act
                            s_last_dm_activity_time = time.time()

                if nlg_commands.poll():
                    command = nlg_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                if tts_commands.poll():
                    command = tts_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                current_time = time.time()

            # stop processes
            aio_commands.send(Command('stop()', 'HUB', 'AIO'))
            vad_commands.send(Command('stop()', 'HUB', 'VAD'))
            asr_commands.send(Command('stop()', 'HUB', 'ASR'))
            slu_commands.send(Command('stop()', 'HUB', 'SLU'))
            dm_commands.send(Command('stop()', 'HUB', 'DM'))
            nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
            tts_commands.send(Command('stop()', 'HUB', 'TTS'))

            # clean connections
            for c in command_connections:
                while c.poll():
                    c.recv()

            for c in non_command_connections:
                while c.poll():
                    c.recv()

            # wait for processes to stop
            # do not join, because in case of exception the join will not be successful
            # aio.join()
            # vad.join()
            # asr.join()
            # slu.join()
            # dm.join()
            # nlg.join()
            # tts.join()
            #cfg['Logging']['session_logger'].join()

        except KeyboardInterrupt:
            print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process(
            ).name
            self.close_event.set()
            return
        except:
            self.cfg['Logging']['system_logger'].exception(
                'Uncaught exception in AHUB process.')
            self.close_event.set()
            raise

        print 'Exiting: %s. Setting close event' % multiprocessing.current_process(
        ).name
        self.close_event.set()
Example #6
0
def run(cfg, max_ncalls):
    try:
        cfg['Logging']['system_logger'].info("Repeat After Me dialogue system\n" + "=" * 120)

        sample_sentences = load_sentences(cfg['RepeatAfterMe']['sentences_file'])

        vio_commands, vio_child_commands = multiprocessing.Pipe()  # used to send commands to VoipIO
        vio_record, vio_child_record = multiprocessing.Pipe()      # I read from this connection recorded audio
        vio_play, vio_child_play = multiprocessing.Pipe()          # I write in audio to be played

        vad_commands, vad_child_commands = multiprocessing.Pipe()   # used to send commands to VAD
        vad_audio_out, vad_child_audio_out = multiprocessing.Pipe() # used to read output audio from VAD

        tts_commands, tts_child_commands = multiprocessing.Pipe()   # used to send commands to TTS
        tts_text_in, tts_child_text_in = multiprocessing.Pipe()     # used to send TTS text

        command_connections = [vio_commands, vad_commands, tts_commands]

        non_command_connections = [vio_record, vio_child_record,
                                   vio_play, vio_child_play,
                                   vad_audio_out, vad_child_audio_out,
                                   tts_text_in, tts_child_text_in]

        close_event = multiprocessing.Event()

        vio = VoipIO(cfg, vio_child_commands, vio_child_record, vio_child_play, close_event)
        vad = VAD(cfg, vad_child_commands, vio_record, vad_child_audio_out, close_event)
        tts = TTS(cfg, tts_child_commands, tts_child_text_in, vio_play, close_event)

        vio.start()
        vad.start()
        tts.start()

        cfg['Logging']['session_logger'].set_close_event(close_event)
        cfg['Logging']['session_logger'].set_cfg(cfg)
        cfg['Logging']['session_logger'].start()

        # init the system
        call_connected = False
        call_start = 0
        count_intro = 0
        intro_played = False
        reject_played = False
        intro_id = 0
        last_intro_id = -1
        end_played = False
        s_voice_activity = False
        s_last_voice_activity_time = 0
        u_voice_activity = False
        u_last_voice_activity_time = 0
        ncalls = 0

        db = load_database(cfg['RepeatAfterMe']['call_db'])

        for remote_uri in db['calls_from_start_end_length']:
            num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

            m = []
            m.append('')
            m.append('=' * 120)
            m.append('Remote SIP URI: %s' % remote_uri)
            m.append('-' * 120)
            m.append('Total calls:             %d' % num_all_calls)
            m.append('Total time (s):          %f' % total_time)
            m.append('Last 24h total calls:    %d' % last24_num_calls)
            m.append('Last 24h total time (s): %f' % last24_total_time)
            m.append('-' * 120)

            current_time = time.time()
            if last24_num_calls > cfg['RepeatAfterMe']['last24_max_num_calls'] or \
                    last24_total_time > cfg['RepeatAfterMe']['last24_max_total_time']:

                # add the remote uri to the black list
                vio_commands.send(Command('black_list(remote_uri="%s",expire="%d")' % (remote_uri,
                                                                                       current_time + cfg['RepeatAfterMe']['blacklist_for']), 'HUB', 'VoipIO'))
                m.append('BLACKLISTED')
            else:
                m.append('OK')

            m.append('-' * 120)
            m.append('')
            cfg['Logging']['system_logger'].info('\n'.join(m))

        call_back_time = -1
        call_back_uri = None

        while 1:
            # Check the close event.
            if close_event.is_set():
                print 'Received close event in: %s' % multiprocessing.current_process().name
                return

            time.sleep(cfg['Hub']['main_loop_sleep_time'])

            if vad_audio_out.poll():
                data_vad = vad_audio_out.recv()

            if call_back_time != -1 and call_back_time < time.time():
                vio_commands.send(Command('make_call(destination="%s")' % call_back_uri, 'HUB', 'VoipIO'))
                call_back_time = -1
                call_back_uri = None

            # read all messages
            if vio_commands.poll():
                command = vio_commands.recv()

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "incoming_call" or command.parsed['__name__'] == "make_call":
                        cfg['Logging']['system_logger'].session_start(command.parsed['remote_uri'])
                        cfg['Logging']['session_logger'].session_start(cfg['Logging']['system_logger'].get_session_dir_name())

                        cfg['Logging']['system_logger'].session_system_log('config = ' + unicode(cfg))
                        cfg['Logging']['system_logger'].info(command)

                        cfg['Logging']['session_logger'].config('config = ' + unicode(cfg))
                        cfg['Logging']['session_logger'].header(cfg['Logging']["system_name"], cfg['Logging']["version"])
                        cfg['Logging']['session_logger'].input_source("voip")

                    if command.parsed['__name__'] == "rejected_call":
                        cfg['Logging']['system_logger'].info(command)

                        call_back_time = time.time() + cfg['RepeatAfterMe']['wait_time_before_calling_back']
                        # call back a default uri, if not defined call back the caller
                        if ('call_back_uri_subs' in cfg['RepeatAfterMe']) and cfg['RepeatAfterMe']['call_back_uri_subs']:
                            ru = command.parsed['remote_uri']
                            for pat, repl in cfg['RepeatAfterMe']['call_back_uri_subs']:
                                ru = re.sub(pat, repl, ru)
                            call_back_uri = ru
                        elif ('call_back_uri' in cfg['RepeatAfterMe']) and cfg['RepeatAfterMe']['call_back_uri']:
                            call_back_uri = cfg['RepeatAfterMe']['call_back_uri']
                        else:
                            call_back_uri = command.parsed['remote_uri']

                    if command.parsed['__name__'] == "rejected_call_from_blacklisted_uri":
                        cfg['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']

                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append('Rejected incoming call from blacklisted URI: %s' % remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' % last24_num_calls)
                        m.append('Last 24h total time (s): %f' % last24_total_time)
                        m.append('=' * 120)
                        m.append('')
                        cfg['Logging']['system_logger'].info('\n'.join(m))

                    if command.parsed['__name__'] == "call_connecting":
                        cfg['Logging']['system_logger'].info(command)

                    if command.parsed['__name__'] == "call_confirmed":
                        cfg['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']
                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append('Incoming call from :     %s' % remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' % last24_num_calls)
                        m.append('Last 24h total time (s): %f' % last24_total_time)
                        m.append('-' * 120)

                        if last24_num_calls > cfg['RepeatAfterMe']['last24_max_num_calls'] or \
                                last24_total_time > cfg['RepeatAfterMe']['last24_max_total_time']:

                            cfg['Logging']['session_logger'].turn("system")
                            tts_commands.send(Command('synthesize(text="%s",log="true")' % cfg['RepeatAfterMe']['rejected'], 'HUB', 'TTS'))
                            call_connected = True
                            reject_played = True
                            s_voice_activity = True
                            vio_commands.send(Command('black_list(remote_uri="%s",expire="%d")' % (remote_uri, time.time() + cfg['RepeatAfterMe']['blacklist_for']), 'HUB', 'VoipIO'))
                            m.append('CALL REJECTED')
                        else:
                            # init the system
                            call_connected = True
                            call_start = time.time()
                            count_intro = 0
                            intro_played = False
                            reject_played = False
                            end_played = False
                            s_voice_activity = False
                            s_last_voice_activity_time = 0
                            u_voice_activity = False
                            u_last_voice_activity_time = 0

                            intro_id, last_intro_id = play_intro(cfg, tts_commands, intro_id, last_intro_id)

                            m.append('CALL ACCEPTED')

                        m.append('=' * 120)
                        m.append('')
                        cfg['Logging']['system_logger'].info('\n'.join(m))

                        try:
                            db['calls_from_start_end_length'][
                                remote_uri].append([time.time(), 0, 0])
                        except:
                            db['calls_from_start_end_length'][
                                remote_uri] = [[time.time(), 0, 0], ]
                        save_database(cfg['RepeatAfterMe']['call_db'], db)

                    if command.parsed['__name__'] == "call_disconnected":
                        cfg['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']

                        vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                        vad_commands.send(Command('flush()', 'HUB', 'VAD'))
                        tts_commands.send(Command('flush()', 'HUB', 'TTS'))

                        cfg['Logging']['system_logger'].session_end()
                        cfg['Logging']['session_logger'].session_end()

                        try:
                            s, e, l = db[
                                'calls_from_start_end_length'][remote_uri][-1]

                            if e == 0 and l == 0:
                                # there is a record about last confirmed but not disconnected call
                                db['calls_from_start_end_length'][remote_uri][-1] = [s, time.time(), time.time() - s]
                                save_database('call_db.pckl', db)
                        except KeyError:
                            # disconnecting call which was not confirmed for URI calling for the first time
                            pass

                        intro_played = False
                        call_connected = False
                        ncalls += 1

                    if command.parsed['__name__'] == "play_utterance_start":
                        cfg['Logging']['system_logger'].info(command)
                        s_voice_activity = True

                    if command.parsed['__name__'] == "play_utterance_end":
                        cfg['Logging']['system_logger'].info(command)

                        s_voice_activity = False
                        s_last_voice_activity_time = time.time()

                        if command.parsed['user_id'] == last_intro_id:
                            intro_played = True
                            s_last_voice_activity_time = 0

            if vad_commands.poll():
                command = vad_commands.recv()
                cfg['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "speech_start":
                        u_voice_activity = True
                    if command.parsed['__name__'] == "speech_end":
                        u_voice_activity = False
                        u_last_voice_activity_time = time.time()

            if tts_commands.poll():
                command = tts_commands.recv()
                cfg['Logging']['system_logger'].info(command)

            current_time = time.time()

        #  print
        #  print intro_played, end_played
        #  print s_voice_activity, u_voice_activity,
        #  print call_start,  current_time, u_last_voice_activity_time, s_last_voice_activity_time
        #  print current_time - s_last_voice_activity_time > 5, u_last_voice_activity_time - s_last_voice_activity_time > 0

            if reject_played == True and s_voice_activity == False:
                # be careful it does not hangup immediately
                reject_played = False
                vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))
                vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                vad_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                tts_commands.send(Command('flush()', 'HUB', 'VoipIO'))

            if intro_played and current_time - call_start > cfg['RepeatAfterMe']['max_call_length'] and s_voice_activity == False:
                # too long call
                if not end_played:
                    s_voice_activity = True
                    last_intro_id = str(intro_id)
                    intro_id += 1
                    cfg['Logging']['session_logger'].turn("system")
                    tts_commands.send(Command('synthesize(text="%s",log="true")' % cfg['RepeatAfterMe']['closing'], 'HUB', 'TTS'))
                    end_played = True
                else:
                    intro_played = False
                    # be careful it does not hangup immediately
                    vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))
                    vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                    vad_commands.send(Command('flush()', 'HUB', 'VAD'))
                    tts_commands.send(Command('flush()', 'HUB', 'TTS'))

            if intro_played and \
                s_voice_activity == False and \
                u_voice_activity == False and \
                current_time - s_last_voice_activity_time > 5 and current_time - u_last_voice_activity_time > 0.6:

                if 'silence' not in cfg['RepeatAfterMe'] or not cfg['RepeatAfterMe']['silence']:
                    s_voice_activity = True

                    s1 = ram()
                    cfg['Logging']['session_logger'].turn("system")
                    tts_commands.send(Command('synthesize(text="%s",log="true")' % s1, 'HUB', 'TTS'))
                    s2 = sample_sentence(sample_sentences)
                    tts_commands.send(Command('synthesize(text="%s",log="true")' % s2, 'HUB', 'TTS'))

                    s = s1 + ' ' + s2

                    m = []
                    m.append('=' * 120)
                    m.append('Say: ' + s)
                    m.append('=' * 120)

                    cfg['Logging']['system_logger'].info('\n'.join(m))

            if ncalls != 0 and not call_connected and s_last_voice_activity_time + 5.0 < current_time and ncalls >= max_ncalls:
                break

        # stop processes
        vio_commands.send(Command('stop()', 'HUB', 'VoipIO'))
        vad_commands.send(Command('stop()', 'HUB', 'VAD'))
        tts_commands.send(Command('stop()', 'HUB', 'TTS'))

        # clean connections
        for c in command_connections:
            while c.poll():
                c.recv()

        for c in non_command_connections:
            while c.poll():
                c.recv()

        # wait for processes to stop
        # do not join, because in case of exception the join will not be successful
        #vio.join()
        #system_logger.debug('VIO stopped.')
        #vad.join()
        #system_logger.debug('VAD stopped.')
        #tts.join()
        #system_logger.debug('TTS stopped.')

    except KeyboardInterrupt:
        print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process().name
        close_event.set()
        return
    except:
        cfg['Logging']['system_logger'].exception('Uncaught exception in SW_HUB process.')
        close_event.set()
        raise

    print 'RAM HUB: Exiting: %s. Setting close event' % multiprocessing.current_process().name
    close_event.set()
Example #7
0
    def run(self):
        try:
            # AIO pipes
            # used to send commands to VoipIO
            aio_commands, aio_child_commands = multiprocessing.Pipe()
            # I read from this connection recorded audio
            aio_record, aio_child_record = multiprocessing.Pipe()
            # I write in audio to be played
            aio_play, aio_child_play = multiprocessing.Pipe()

            # VAD pipes
            # used to send commands to VAD
            vad_commands, vad_child_commands = multiprocessing.Pipe()
            # used to read output audio from VAD
            vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()

            # ASR pipes
            # used to send commands to ASR
            asr_commands, asr_child_commands = multiprocessing.Pipe()
            # used to read ASR hypotheses
            asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()

            # SLU pipes
            # used to send commands to SLU
            slu_commands, slu_child_commands = multiprocessing.Pipe()
            # used to read SLU hypotheses
            slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()

            # DM pipes
            # used to send commands to DM
            dm_commands, dm_child_commands = multiprocessing.Pipe()
            # used to read DM actions
            dm_actions_out, dm_child_actions = multiprocessing.Pipe()

            # NLG pipes
            # used to send commands to NLG
            nlg_commands, nlg_child_commands = multiprocessing.Pipe()
            # used to read NLG output
            nlg_text_out, nlg_child_text = multiprocessing.Pipe()

            # TTS pipes
            # used to send commands to TTS
            tts_commands, tts_child_commands = multiprocessing.Pipe()


            command_connections = [aio_commands, vad_commands, asr_commands,
                                   slu_commands, dm_commands, nlg_commands,
                                   tts_commands]

            non_command_connections = [aio_record, aio_child_record,
                                       aio_play, aio_child_play,
                                       vad_audio_out, vad_child_audio_out,
                                       asr_hypotheses_out, asr_child_hypotheses,
                                       slu_hypotheses_out, slu_child_hypotheses,
                                       dm_actions_out, dm_child_actions,
                                       nlg_text_out, nlg_child_text]

            self.close_event = multiprocessing.Event()

            # create the hub components
            aio = AudioIO(self.cfg, aio_child_commands, aio_child_record, aio_child_play, self.close_event)
            vad = VAD(self.cfg, vad_child_commands, aio_record, vad_child_audio_out, self.close_event)
            asr = ASR(self.cfg, asr_child_commands, vad_audio_out, asr_child_hypotheses, self.close_event)
            slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out, slu_child_hypotheses, self.close_event)
            dm  =  DM(self.cfg,  dm_child_commands, slu_hypotheses_out, dm_child_actions, self.close_event)
            nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out, nlg_child_text, self.close_event)
            tts = TTS(self.cfg, tts_child_commands, nlg_text_out, aio_play, self.close_event)

            # start the hub components
            aio.start()
            vad.start()
            asr.start()
            slu.start()
            dm.start()
            nlg.start()
            tts.start()

            self.cfg['Logging']['session_logger'].set_close_event(self.close_event)
            self.cfg['Logging']['session_logger'].set_cfg(self.cfg)
            self.cfg['Logging']['session_logger'].start()

            # init the system
            call_start = 0
            call_back_time = -1
            call_back_uri = None

            s_voice_activity = False
            s_last_voice_activity_time = 0
            u_voice_activity = False
            u_last_voice_activity_time = 0

            s_last_dm_activity_time = 0

            hangup = False

            call_start = time.time()

            self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
            self.cfg['Logging']['system_logger'].session_system_log('config = ' + str(self.cfg))

            self.cfg['Logging']['session_logger'].session_start(self.cfg['Logging']['system_logger'].get_session_dir_name())
            self.cfg['Logging']['session_logger'].config('config = ' + str(self.cfg))
            self.cfg['Logging']['session_logger'].header(self.cfg['Logging']["system_name"], self.cfg['Logging']["version"])
            self.cfg['Logging']['session_logger'].input_source("aio")


            while 1:
                # Check the close event.
                if self.close_event.is_set():
                    print 'Received close event in: %s' % multiprocessing.current_process().name
                    return

                time.sleep(self.cfg['Hub']['main_loop_sleep_time'])

                if call_back_time != -1 and call_back_time < time.time():
                    aio_commands.send(Command('make_call(destination="%s")' % \
                                              call_back_uri, 'HUB', 'AIO'))
                    call_back_time = -1
                    call_back_uri = None

                if vad_commands.poll():
                    command = vad_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "speech_start":
                            u_voice_activity = True
                        if command.parsed['__name__'] == "speech_end":
                            u_voice_activity = False
                            u_last_voice_activity_time = time.time()

                if asr_commands.poll():
                    command = asr_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                if slu_commands.poll():
                    command = slu_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                if dm_commands.poll():
                    command = dm_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "hangup":
                            # prepare for ending the call
                            hangup = True

                        if command.parsed['__name__'] == "dm_da_generated":
                            # record the time of the last system generated dialogue act
                            s_last_dm_activity_time = time.time()

                if nlg_commands.poll():
                    command = nlg_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                if tts_commands.poll():
                    command = tts_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                current_time = time.time()

            # stop processes
            aio_commands.send(Command('stop()', 'HUB', 'AIO'))
            vad_commands.send(Command('stop()', 'HUB', 'VAD'))
            asr_commands.send(Command('stop()', 'HUB', 'ASR'))
            slu_commands.send(Command('stop()', 'HUB', 'SLU'))
            dm_commands.send(Command('stop()', 'HUB', 'DM'))
            nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
            tts_commands.send(Command('stop()', 'HUB', 'TTS'))

            # clean connections
            for c in command_connections:
                while c.poll():
                    c.recv()

            for c in non_command_connections:
                while c.poll():
                    c.recv()

            # wait for processes to stop
            # do not join, because in case of exception the join will not be successful
            # aio.join()
            # vad.join()
            # asr.join()
            # slu.join()
            # dm.join()
            # nlg.join()
            # tts.join()
            #cfg['Logging']['session_logger'].join()

        except KeyboardInterrupt:
            print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process().name
            self.close_event.set()
            return
        except:
            self.cfg['Logging']['system_logger'].exception('Uncaught exception in AHUB process.')
            self.close_event.set()
            raise

        print 'Exiting: %s. Setting close event' % multiprocessing.current_process().name
        self.close_event.set()
Example #8
0
def run(cfg1, cfg2):
    try:
        cfg1['Logging']['system_logger'].info("Switchboard system\n" + "=" * 120)

        vio1_commands, vio1_child_commands = multiprocessing.Pipe()  # used to send commands to VoipIO
        vio1_record, vio1_child_record = multiprocessing.Pipe()      # I read from this connection recorded audio
        vio1_play, vio1_child_play = multiprocessing.Pipe()          # I write in audio to be played

        vad1_commands, vad1_child_commands = multiprocessing.Pipe()   # used to send commands to VAD
        vad1_audio_in, vad1_child_audio_in = multiprocessing.Pipe() # used to read output audio from VAD
        vad1_audio_out, vad1_child_audio_out = multiprocessing.Pipe() # used to read output audio from VAD

        tts1_commands, tts1_child_commands = multiprocessing.Pipe()   # used to send commands to TTS
        tts1_text_in, tts1_child_text_in = multiprocessing.Pipe()     # used to send TTS text

        vio2_commands, vio2_child_commands = multiprocessing.Pipe()  # used to send commands to VoipIO
        vio2_record, vio2_child_record = multiprocessing.Pipe()      # I read from this connection recorded audio
        vio2_play, vio2_child_play = multiprocessing.Pipe()          # I write in audio to be played

        vad2_commands, vad2_child_commands = multiprocessing.Pipe()   # used to send commands to VAD
        vad2_audio_in, vad2_child_audio_in = multiprocessing.Pipe() # used to read output audio from VAD
        vad2_audio_out, vad2_child_audio_out = multiprocessing.Pipe() # used to read output audio from VAD

        tts2_commands, tts2_child_commands = multiprocessing.Pipe()   # used to send commands to TTS
        tts2_text_in, tts2_child_text_in = multiprocessing.Pipe()     # used to send TTS text

        command_connections = [vio1_commands, vad1_commands, tts1_commands, vio2_commands, vad2_commands, tts2_commands]

        non_command_connections = [vio1_record, vio1_child_record,
                                   vio1_play, vio1_child_play,
                                   vad1_audio_in, vad1_child_audio_in,
                                   vad1_audio_out, vad1_child_audio_out,
                                   tts1_text_in, tts1_child_text_in,
                                   vio2_record, vio2_child_record,
                                   vio2_play, vio2_child_play,
                                   vad2_audio_in, vad2_child_audio_in,
                                   vad2_audio_out, vad2_child_audio_out,
                                   tts2_text_in, tts2_child_text_in]

        close_event = multiprocessing.Event()

        vio1 = VoipIO(cfg1, vio1_child_commands, vio1_child_record, vio1_child_play, close_event)
        vad1 = VAD(cfg1, vad1_child_commands, vad1_child_audio_in, vad1_child_audio_out, close_event)
        tts1 = TTS(cfg1, tts1_child_commands, tts1_child_text_in, vio1_play, close_event)
        vio2 = VoipIO(cfg2, vio2_child_commands, vio2_child_record, vio2_child_play, close_event)
        vad2 = VAD(cfg2, vad2_child_commands, vad2_child_audio_in, vad2_child_audio_out, close_event)
        tts2 = TTS(cfg2, tts2_child_commands, tts2_child_text_in, vio2_play, close_event)


        vio1.start()
        vad1.start()
        tts1.start()

        vio2.start()
        vad2.start()
        tts2.start()

        cfg1['Logging']['session_logger'].set_close_event(close_event)
        cfg1['Logging']['session_logger'].set_cfg(cfg1)
        cfg1['Logging']['session_logger'].start()
        cfg1['Logging']['session_logger'].cancel_join_thread()

        cfg2['Logging']['session_logger'].set_close_event(close_event)
        cfg2['Logging']['session_logger'].set_cfg(cfg2)
        cfg2['Logging']['session_logger'].start()
        cfg2['Logging']['session_logger'].cancel_join_thread()

        # init the system
        call_start1 = 0
        count_intro1 = 0
        intro_played1 = False
        reject_played1 = False
        intro_id1 = 0
        last_intro_id1 = -1
        end_played1 = False
        s_voice_activity1 = False
        s_last_voice_activity_time1 = 0
        u_voice_activity1 = False
        u_last_voice_activity_time1 = 0
        vio_connect1 = False
        hangup1 = False
        u_start1 = False

        call_start2 = 0
        count_intro2 = 0
        intro_played2 = False
        reject_played2 = False
        intro_id2 = 0
        last_intro_id2 = -1
        end_played2 = False
        s_voice_activity2 = False
        s_last_voice_activity_time2 = 0
        u_voice_activity2 = False
        u_last_voice_activity_time2 = 0
        vio_connect2 = False
        hangup2 = False
        u_start2 = False

        callee_entered = False
        callee_uri = ''

        db = load_database(cfg1['Switchboard']['call_db'])

        for remote_uri in db['calls_from_start_end_length']:
            num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

            m = []
            m.append('')
            m.append('=' * 120)
            m.append('Remote SIP URI: %s' % remote_uri)
            m.append('-' * 120)
            m.append('Total calls:             %d' % num_all_calls)
            m.append('Total time (s):          %f' % total_time)
            m.append('Last 24h total calls:    %d' % last24_num_calls)
            m.append('Last 24h total time (s): %f' % last24_total_time)
            m.append('-' * 120)

            current_time = time.time()
            if last24_num_calls > cfg1['Switchboard']['last24_max_num_calls'] or \
                    last24_total_time > cfg1['Switchboard']['last24_max_total_time']:

                # add the remote uri to the black list
                vio1_commands.send(Command('black_list(remote_uri="%s",expire="%d")' % (remote_uri,
                    current_time + cfg1['Switchboard']['blacklist_for']), 'HUB', 'VoipIO'))
                m.append('BLACKLISTED')
            else:
                m.append('OK')

            m.append('-' * 120)
            m.append('')
            cfg1['Logging']['system_logger'].info('\n'.join(m))

        call_back_time = -1
        call_back_uri = None

        while 1:
            # Check the close event.
            if close_event.is_set():
                print 'Received close event in: %s' % multiprocessing.current_process().name
                return

            time.sleep(cfg1['Hub']['main_loop_sleep_time'])

            if intro_played1 and intro_played2 and not u_start1:
                vio1_commands.send(Command('flush_out()', 'HUB', 'VoipIO1'))
                time.sleep(cfg1['Hub']['main_loop_sleep_time'])
                cfg1['Logging']['session_logger'].turn("system")
                vio1_play.send(Command('utterance_start(user_id="%s",text="%s",fname="%s",log="%s")' %
                    ('system', '', 'vpl-1.wav', 'true'), 'HUB', 'VoipIO1'))
                u_start1 = True

            if intro_played1 and intro_played2 and not u_start2:
                vio2_commands.send(Command('flush_out()', 'HUB', 'VoipIO2'))
                time.sleep(cfg1['Hub']['main_loop_sleep_time'])
                cfg2['Logging']['session_logger'].turn("system")
                vio2_play.send(Command('utterance_start(user_id="%s",text="%s",fname="%s",log="%s")' %
                    ('system', '', 'vpl-2.wav', 'true'), 'HUB', 'VoipIO2'))
                u_start2 = True

            while vio1_record.poll() or vio2_record.poll():
                if vio1_record.poll():
                    data = vio1_record.recv()
                    vad1_audio_in.send(data)

                    if u_start2:
                        vio2_play.send(data)

                if vio2_record.poll():
                    data = vio2_record.recv()
                    vad2_audio_in.send(data)

                    if u_start1:
                        vio1_play.send(data)

            # empty vad pipes
            while vad1_audio_out.poll():
                data = vad1_audio_out.recv()
            while vad2_audio_out.poll():
                data = vad2_audio_out.recv()

            if call_back_time != -1 and call_back_time < time.time():
                vio1_commands.send(Command('make_call(destination="%s")' % call_back_uri, 'HUB', 'VoipIO1'))

                call_back_time = -1
                call_back_uri = None

            if callee_entered and callee_uri:
                s_voice_activity1 = True
                m = cfg1['Switchboard']['calling'] + ' '.join(callee_uri)
                cfg1['Logging']['session_logger'].turn("system")
                tts1_commands.send(Command('synthesize(text="%s",log="true")' % m, 'HUB', 'TTS1'))

                vio2_commands.send(Command('make_call(destination="%s")' % callee_uri, 'HUB', 'VoipIO2'))

                callee_uri = ''

            # read all messages
            if vio1_commands.poll():
                command = vio1_commands.recv()

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "incoming_call" or command.parsed['__name__'] == "make_call":
                        cfg1['Logging']['system_logger'].session_start(command.parsed['remote_uri'])
                        cfg1['Logging']['session_logger'].session_start(cfg1['Logging']['system_logger'].get_session_dir_name())

                        cfg1['Logging']['system_logger'].session_system_log('config = ' + unicode(cfg1))
                        cfg1['Logging']['system_logger'].info(command)

                        cfg1['Logging']['session_logger'].config('config = ' + unicode(cfg1))
                        cfg1['Logging']['session_logger'].header(cfg1['Logging']["system_name"], cfg1['Logging']["version"])
                        cfg1['Logging']['session_logger'].input_source("voip")

                    if command.parsed['__name__'] == "rejected_call":
                        cfg1['Logging']['system_logger'].info(command)

                        call_back_time = time.time() + cfg1['Switchboard']['wait_time_before_calling_back']
                        # call back a default uri, if not defined call back the caller
                        if ('call_back_uri_subs' in cfg1['Switchboard']) and cfg1['Switchboard']['call_back_uri_subs']:
                            ru = command.parsed['remote_uri']
                            for pat, repl in cfg1['Switchboard']['call_back_uri_subs']:
                                ru = re.sub(pat, repl, ru)
                            call_back_uri = ru
                        elif ('call_back_uri' in cfg1['Switchboard']) and cfg1['Switchboard']['call_back_uri']:
                            call_back_uri = cfg1['Switchboard']['call_back_uri']
                        else:
                            call_back_uri = command.parsed['remote_uri']

                    if command.parsed['__name__'] == "rejected_call_from_blacklisted_uri":
                        cfg1['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']

                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append('Rejected incoming call from blacklisted URI: %s' % remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' % last24_num_calls)
                        m.append('Last 24h total time (s): %f' % last24_total_time)
                        m.append('=' * 120)
                        m.append('')
                        cfg1['Logging']['system_logger'].info('\n'.join(m))

                    if command.parsed['__name__'] == "call_connecting":
                        cfg1['Logging']['system_logger'].info(command)

                    if command.parsed['__name__'] == "call_confirmed":
                        cfg1['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']
                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append('Incoming call from :     %s' % remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' % last24_num_calls)
                        m.append('Last 24h total time (s): %f' % last24_total_time)
                        m.append('-' * 120)

                        if last24_num_calls > cfg1['Switchboard']['last24_max_num_calls'] or \
                                last24_total_time > cfg1['Switchboard']['last24_max_total_time']:

                            cfg1['Logging']['session_logger'].turn("system")
                            tts1_commands.send(Command('synthesize(text="%s",log="true")' % cfg1['Switchboard']['rejected'], 'HUB', 'TTS1'))
                            reject_played1 = True
                            s_voice_activity1 = True
                            vio1_commands.send(Command('black_list(remote_uri="%s",expire="%d")' % (remote_uri, time.time() + cfg1['Switchboard']['blacklist_for']), 'HUB', 'VoipIO1'))
                            m.append('CALL REJECTED')
                        else:
                            # init the system
                            call_start1 = time.time()
                            count_intro1 = 0
                            intro_played1 = False
                            reject_played1 = False
                            end_played1 = False
                            s_voice_activity1 = False
                            s_last_voice_activity_time1 = 0
                            u_voice_activity1 = False
                            u_last_voice_activity_time1 = 0
                            vio_connect1 = False
                            hangup1 = False
                            u_start1 = False

                            callee_entered = False
                            callee_uri = ''

                            intro_id1, last_intro_id1 = play_intro(cfg1, tts1_commands, intro_id1, last_intro_id1)

                            m.append('CALL ACCEPTED')

                        m.append('=' * 120)
                        m.append('')
                        cfg1['Logging']['system_logger'].info('\n'.join(m))

                        try:
                            db['calls_from_start_end_length'][remote_uri].append([time.time(), 0, 0])
                        except:
                            db['calls_from_start_end_length'][remote_uri] = [[time.time(), 0, 0], ]
                        save_database(cfg1['Switchboard']['call_db'], db)

                    if command.parsed['__name__'] == "call_disconnected":
                        cfg1['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']

                        vio1_commands.send(Command('flush()', 'HUB', 'VoipIO1'))
                        vad1_commands.send(Command('flush()', 'HUB', 'VAD1'))
                        tts1_commands.send(Command('flush()', 'HUB', 'TTS1'))

                        cfg1['Logging']['system_logger'].session_end()
                        cfg1['Logging']['session_logger'].session_end()

                        try:
                            s, e, l = db['calls_from_start_end_length'][remote_uri][-1]

                            if e == 0 and l == 0:
                                # there is a record about last confirmed but not disconnected call
                                db['calls_from_start_end_length'][remote_uri][-1] = [s, time.time(), time.time() - s]
                                save_database('call_db.pckl', db)
                        except KeyError:
                            # disconnecting call which was not confirmed for URI calling for the first time
                            pass

                        intro_played1 = False
                        u_start1 = False

                        callee_entered = False
                        callee_uri = ''

                        vio2_commands.send(Command('hangup()', 'HUB', 'VoipIO1'))

                    if command.parsed['__name__'] == "play_utterance_start":
                        cfg1['Logging']['system_logger'].info(command)
                        s_voice_activity1 = True

                    if command.parsed['__name__'] == "play_utterance_end":
                        cfg1['Logging']['system_logger'].info(command)

                        s_voice_activity1 = False
                        s_last_voice_activity_time1 = time.time()

                        if command.parsed['user_id'] == last_intro_id1:
                            intro_played1 = True

                    if command.parsed['__name__'] == "dtmf_digit":
                        cfg1['Logging']['system_logger'].info(command)

                        digit = command.parsed['digit']

                        if digit in ['*', '#']:
                            callee_entered = True

                        if not callee_entered and digit in '0123456789':
                            callee_uri += digit

            if vio2_commands.poll():
                command = vio2_commands.recv()

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "make_call":
                        cfg2['Logging']['system_logger'].session_start(command.parsed['remote_uri'])
                        cfg2['Logging']['session_logger'].session_start(cfg2['Logging']['system_logger'].get_session_dir_name())

                        cfg2['Logging']['system_logger'].session_system_log('config = ' + unicode(cfg2))
                        cfg2['Logging']['system_logger'].info(command)

                        cfg2['Logging']['session_logger'].config('config = ' + unicode(cfg2))
                        cfg2['Logging']['session_logger'].header(cfg2['Logging']["system_name"], cfg2['Logging']["version"])
                        cfg2['Logging']['session_logger'].input_source("voip")

                    if command.parsed['__name__'] == "call_connecting":
                        cfg2['Logging']['system_logger'].info(command)

                    if command.parsed['__name__'] == "call_confirmed":
                        cfg2['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']
                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append('Incoming call from :     %s' % remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' % last24_num_calls)
                        m.append('Last 24h total time (s): %f' % last24_total_time)
                        m.append('-' * 120)

                        m.append('CALL ACCEPTED')

                        m.append('=' * 120)
                        m.append('')
                        cfg2['Logging']['system_logger'].info('\n'.join(m))

                        # init the system
                        call_start2 = time.time()
                        count_intro2 = 0
                        intro_played2 = False
                        reject_played2 = False
                        end_played2 = False
                        s_voice_activity2 = False
                        s_last_voice_activity_time2 = 0
                        u_voice_activity2 = False
                        u_last_voice_activity_time2 = 0
                        vio_connect2 = False
                        hangup2 = False
                        u_start2 = False

                        intro_id2, last_intro_id2 = play_intro(cfg2, tts2_commands, intro_id2, last_intro_id2)

                    if command.parsed['__name__'] == "call_disconnected":
                        cfg2['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']
                        code = command.parsed['code']

                        vio2_commands.send(Command('flush()', 'HUB', 'VoipIO2'))
                        vad2_commands.send(Command('flush()', 'HUB', 'VAD2'))
                        tts2_commands.send(Command('flush()', 'HUB', 'TTS2'))

                        cfg2['Logging']['system_logger'].session_end()
                        cfg2['Logging']['session_logger'].session_end()

                        if intro_played2 and code in set(['603',]):
                            vio1_commands.send(Command('hangup()', 'HUB', 'VoipIO1'))
                        elif code in set(['486', '600', '603', '604', '606',]):
                            s_voice_activity1 = True
                            m = cfg1['Switchboard']['noanswer']
                            cfg1['Logging']['session_logger'].turn("system")
                            tts1_commands.send(Command('synthesize(text="%s",log="true")' % m, 'HUB', 'TTS1'))
                        elif code in set(['480',]):
                            s_voice_activity1 = True
                            m = cfg1['Switchboard']['wrongnumber']
                            cfg1['Logging']['session_logger'].turn("system")
                            tts1_commands.send(Command('synthesize(text="%s",log="true")' % m, 'HUB', 'TTS1'))
                        else:
                            vio1_commands.send(Command('hangup()', 'HUB', 'VoipIO1'))

                        intro_played2 = False
                        u_start2 = False
                        hangup1 = True

                    if command.parsed['__name__'] == "play_utterance_start":
                        cfg2['Logging']['system_logger'].info(command)
                        s_voice_activity2 = True

                    if command.parsed['__name__'] == "play_utterance_end":
                        cfg2['Logging']['system_logger'].info(command)

                        s_voice_activity2 = False
                        s_last_voice_activity_time2 = time.time()

                        if command.parsed['user_id'] == last_intro_id2:
                            intro_played2 = True

            if vad1_commands.poll():
                command = vad1_commands.recv()
                cfg1['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "speech_start":
                        u_voice_activity = True
                    if command.parsed['__name__'] == "speech_end":
                        u_voice_activity = False
                        u_last_voice_activity_time = time.time()

            if vad2_commands.poll():
                command = vad2_commands.recv()
                cfg2['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "speech_start":
                        u_voice_activity = True
                    if command.parsed['__name__'] == "speech_end":
                        u_voice_activity = False
                        u_last_voice_activity_time = time.time()

            if tts1_commands.poll():
                command = tts1_commands.recv()
                cfg1['Logging']['system_logger'].info(command)

            if tts2_commands.poll():
                command = tts2_commands.recv()
                cfg1['Logging']['system_logger'].info(command)

            current_time = time.time()

            # print
            # print intro_played, end_played
            # print s_voice_activity, u_voice_activity,
            # print call_start,  current_time, u_last_voice_activity_time, s_last_voice_activity_time
            # print current_time - s_last_voice_activity_time > 5, u_last_voice_activity_time - s_last_voice_activity_time > 0
            # print hangup1, s_voice_activity1, s_last_voice_activity_time1, current_time

            if hangup1 and s_voice_activity1 == False and s_last_voice_activity_time1 + 2.0 < current_time:
                # we are ready to hangup only when all voice activity is finished
                hangup1 = False
                vio1_commands.send(Command('hangup()', 'HUB', 'VoipIO1'))

            if hangup2 and s_voice_activity2 == False and s_last_voice_activity_time2 + 2.0 < current_time:
                # we are ready to hangup only when all voice activity is finished
                hangup2 = False
                vio2_commands.send(Command('hangup()', 'HUB', 'VoipIO2'))

            if reject_played1 == True and s_voice_activity1 == False:
                # be careful it does not hangup immediately
                reject_played1 = False
                vio1_commands.send(Command('hangup()', 'HUB', 'VoipIO1'))
                vio1_commands.send(Command('flush()', 'HUB', 'VoipIO1'))
                vad1_commands.send(Command('flush()', 'HUB', 'VAD1'))
                tts1_commands.send(Command('flush()', 'HUB', 'TTS1'))

            if intro_played1 and current_time - call_start1 > cfg1['Switchboard']['max_call_length']:
                intro_played1 = False
                # be careful it does not hangup immediately
                vio1_commands.send(Command('hangup()', 'HUB', 'VoipIO1'))
                vio1_commands.send(Command('flush()', 'HUB', 'VoipIO1'))
                vad1_commands.send(Command('flush()', 'HUB', 'VAD1'))
                tts1_commands.send(Command('flush()', 'HUB', 'TTS1'))

            if intro_played2 and current_time - call_start1 > cfg1['Switchboard']['max_call_length']:
                intro_played2 = False
                # be careful it does not hangup immediately
                vio2_commands.send(Command('hangup()', 'HUB', 'VoipIO2'))
                vio2_commands.send(Command('flush()', 'HUB', 'VoipIO2'))
                vad2_commands.send(Command('flush()', 'HUB', 'VAD2'))
                tts2_commands.send(Command('flush()', 'HUB', 'TTS2'))

        # stop processes
        vio1_commands.send(Command('stop()', 'HUB', 'VoipIO1'))
        vad1_commands.send(Command('stop()', 'HUB', 'VAD1'))
        tts1_commands.send(Command('stop()', 'HUB', 'TTS1'))
        vio2_commands.send(Command('stop()', 'HUB', 'VoipIO2'))
        vad2_commands.send(Command('stop()', 'HUB', 'VAD2'))
        tts2_commands.send(Command('stop()', 'HUB', 'TTS2'))

        # clean connections
        for c in command_connections:
            while c.poll():
                c.recv()

        for c in non_command_connections:
            while c.poll():
                c.recv()

        # wait for processes to stop
        # do not join, because in case of exception the join will not be successful
        # vio1.join()
        # vad1.join()
        # tts1.join()
        # vio2.join()
        # vad2.join()
        # tts2.join()

    except KeyboardInterrupt:
        print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process().name
        close_event.set()
        return
    except:
        cfg1['Logging']['system_logger'].exception('Uncaught exception in SW_HUB process.')
        close_event.set()
        raise

    print 'Exiting: %s. Setting close event' % multiprocessing.current_process().name
    close_event.set()
Example #9
0
File: vhub.py Project: choko/alex
    def run(self):
        try:
            vio_commands, vio_child_commands = multiprocessing.Pipe()  # used to send commands to VoipIO
            vio_record, vio_child_record = multiprocessing.Pipe()      # I read from this connection recorded audio
            vio_play, vio_child_play = multiprocessing.Pipe()          # I write in audio to be played

            vad_commands, vad_child_commands = multiprocessing.Pipe()   # used to send commands to VAD
            vad_audio_out, vad_child_audio_out = multiprocessing.Pipe() # used to read output audio from VAD

            asr_commands, asr_child_commands = multiprocessing.Pipe()          # used to send commands to ASR
            asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()  # used to read ASR hypotheses

            slu_commands, slu_child_commands = multiprocessing.Pipe()          # used to send commands to SLU
            slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()  # used to read SLU hypotheses

            dm_commands, dm_child_commands = multiprocessing.Pipe()            # used to send commands to DM
            dm_actions_out, dm_child_actions = multiprocessing.Pipe()          # used to read DM actions

            nlg_commands, nlg_child_commands = multiprocessing.Pipe()          # used to send commands to NLG
            nlg_text_out, nlg_child_text = multiprocessing.Pipe()              # used to read NLG output

            tts_commands, tts_child_commands = multiprocessing.Pipe()          # used to send commands to TTS

            command_connections = [vio_commands, vad_commands, asr_commands, slu_commands,
                                   dm_commands, nlg_commands, tts_commands]

            non_command_connections = [vio_record, vio_child_record,
                                       vio_play, vio_child_play,
                                       vad_audio_out, vad_child_audio_out,
                                       asr_hypotheses_out, asr_child_hypotheses,
                                       slu_hypotheses_out, slu_child_hypotheses,
                                       dm_actions_out, dm_child_actions,
                                       nlg_text_out, nlg_child_text]

            vio = VoipIO(self.cfg, vio_child_commands, vio_child_record, vio_child_play, self.close_event)
            vad = VAD(self.cfg, vad_child_commands, vio_record, vad_child_audio_out, self.close_event)
            asr = ASR(self.cfg, asr_child_commands, vad_audio_out, asr_child_hypotheses, self.close_event)
            slu = SLU(self.cfg, slu_child_commands, asr_hypotheses_out, slu_child_hypotheses, self.close_event)
            dm  =  DM(self.cfg,  dm_child_commands, slu_hypotheses_out, dm_child_actions, self.close_event)
            nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out, nlg_child_text, self.close_event)
            tts = TTS(self.cfg, tts_child_commands, nlg_text_out, vio_play, self.close_event)

            vio.start()
            vad.start()
            asr.start()
            slu.start()
            dm.start()
            nlg.start()
            tts.start()

            self.write_pid_file([['vio', vio.pid], ['vad', vad.pid], ['asr', asr.pid],
                                 ['slu', slu.pid], ['dm', dm.pid], ['nlg', nlg.pid], ['tts', tts.pid]])

            cfg['Logging']['session_logger'].set_close_event(self.close_event)
            cfg['Logging']['session_logger'].set_cfg(cfg)
            cfg['Logging']['session_logger'].start()
            cfg['Logging']['session_logger'].cancel_join_thread()

            # init the system
            call_start = 0
            call_back_time = -1
            call_back_uri = None
            number_of_turns = -1

            s_voice_activity = False
            s_last_voice_activity_time = 0
            u_voice_activity = False
            u_last_voice_activity_time = 0

            s_last_dm_activity_time = 0

            u_last_input_timeout = 0

            call_connected = False
            hangup = False
            ncalls = 0

            outstanding_nlg_da = None

            call_db = CallDB(self.cfg, self.cfg['VoipHub']['call_db'], self.cfg['VoipHub']['period'])
            #call_db.log()

            while 1:
                # Check the close event.
                if self.close_event.is_set():
                    print 'Received close event in: %s' % multiprocessing.current_process().name
                    return

                time.sleep(self.cfg['Hub']['main_loop_sleep_time'])

                if call_back_time != -1 and call_back_time < time.time():
                    vio_commands.send(Command('make_call(destination="%s")' % call_back_uri, 'HUB', 'VoipIO'))
                    call_back_time = -1
                    call_back_uri = None

                # read all messages
                if vio_commands.poll():
                    command = vio_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):

                        if command.parsed['__name__'] == "incoming_call":
                            self.cfg['Logging']['system_logger'].session_start(command.parsed['remote_uri'])
                            self.cfg['Logging']['session_logger'].session_start(self.cfg['Logging']['system_logger'].get_session_dir_name())

                            self.cfg['Logging']['system_logger'].session_system_log('config = ' + unicode(self.cfg))
                            self.cfg['Logging']['session_logger'].config('config = ' + unicode(self.cfg))
                            self.cfg['Logging']['session_logger'].header(self.cfg['Logging']["system_name"], self.cfg['Logging']["version"])
                            self.cfg['Logging']['session_logger'].input_source("voip")

                            self.cfg['Analytics'].start_session(command.parsed['remote_uri'])
                            self.cfg['Analytics'].track_event('vhub', 'incoming_call', command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "rejected_call":
                            call_back_time = time.time() + self.cfg['VoipHub']['wait_time_before_calling_back']
                            call_back_uri = command.parsed['remote_uri']

                            self.cfg['Analytics'].track_event('vhub', 'rejected_call', command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "rejected_call_from_blacklisted_uri":
                            remote_uri = command.parsed['remote_uri']

                            num_all_calls, total_time, last_period_num_calls, last_period_total_time, last_period_num_short_calls = call_db.get_uri_stats(remote_uri)

                            m = []
                            m.append('')
                            m.append('=' * 120)
                            m.append('Rejected incoming call from blacklisted URI: %s' % remote_uri)
                            m.append('-' * 120)
                            m.append('Total calls:                  %d' % num_all_calls)
                            m.append('Total time (min):             %0.1f' % (total_time/60.0, ))
                            m.append('Last period short calls:      %d' % last_period_num_short_calls)
                            m.append('Last period total calls:      %d' % last_period_num_calls)
                            m.append('Last period total time (min): %0.1f' % (last_period_total_time/60.0, ))
                            m.append('=' * 120)
                            m.append('')
                            self.cfg['Logging']['system_logger'].info('\n'.join(m))

                            self.cfg['Analytics'].track_event('vhub', 'rejected_call_from_blacklisted_uri', command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "call_connecting":
                            self.cfg['Analytics'].track_event('vhub', 'call_connecting', command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "call_confirmed":
                            remote_uri = command.parsed['remote_uri']
                            num_all_calls, total_time, last_period_num_calls, last_period_total_time, last_period_num_short_calls  = call_db.get_uri_stats(remote_uri)

                            m = []
                            m.append('')
                            m.append('=' * 120)
                            m.append('Incoming call from :          %s' % remote_uri)
                            m.append('-' * 120)
                            m.append('Total calls:                  %d' % num_all_calls)
                            m.append('Total time (min):             %0.1f' % (total_time/60.0, ))
                            m.append('Last period short calls:      %d' % last_period_num_short_calls)
                            m.append('Last period total calls:      %d' % last_period_num_calls)
                            m.append('Last period total time (min): %0.1f' % (last_period_total_time/60.0, ))
                            m.append('-' * 120)

                            if last_period_num_calls > self.cfg['VoipHub']['last_period_max_num_calls'] or \
                                    last_period_total_time > self.cfg['VoipHub']['last_period_max_total_time'] or \
                                    last_period_num_short_calls > self.cfg['VoipHub']['last_period_max_num_short_calls'] :
                                # prepare for ending the call
                                call_connected = True

                                call_start = time.time()
                                number_of_turns = -1
                                s_voice_activity = True
                                s_last_voice_activity_time = time.time()
                                u_voice_activity = False
                                u_last_voice_activity_time = time.time()
                                u_last_input_timeout = time.time()
                                hangup = True

                                self.cfg['Logging']['session_logger'].turn("system")
                                tts_commands.send(Command('synthesize(text="%s",log="true")' % self.cfg['VoipHub']['limit_reached_message'], 'HUB', 'TTS'))
                                vio_commands.send(Command('black_list(remote_uri="%s",expire="%d")' % (remote_uri,
                                  time.time() + self.cfg['VoipHub']['blacklist_for']), 'HUB', 'VoipIO'))
                                m.append('CALL REJECTED')
                            else:
                                # init the system
                                call_connected = True

                                call_start = time.time()
                                number_of_turns = 0

                                s_voice_activity = False
                                s_last_voice_activity_time = 0
                                u_voice_activity = False
                                u_last_voice_activity_time = time.time()
                                u_last_input_timeout = time.time()
                                hangup = False

                                dm_commands.send(Command('new_dialogue()', 'HUB', 'DM'))
                                m.append('CALL ACCEPTED')

                            m.append('=' * 120)
                            m.append('')
                            self.cfg['Logging']['system_logger'].info('\n'.join(m))

                            call_db.track_confirmed_call(remote_uri)
                            self.cfg['Analytics'].track_event('vhub', 'call_confirmed', command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "call_disconnected":
                            # flush vio, when flushed, vad will be flushed
                            vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))

                            self.cfg['Logging']['system_logger'].session_end()
                            self.cfg['Logging']['session_logger'].session_end()

                            remote_uri = command.parsed['remote_uri']
                            call_db.track_disconnected_call(remote_uri)

                            call_connected = False
                            number_of_turns = -1
                            ncalls += 1
                            
                            self.cfg['Analytics'].track_event('vhub', 'call_disconnected', command.parsed['remote_uri'])

                        if command.parsed['__name__'] == "play_utterance_start":
                            s_voice_activity = True
                            s_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "play_utterance_end":
                            s_voice_activity = False
                            s_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "flushed":
                            # flush vad, when flushed, asr will be flushed
                            vad_commands.send(Command('flush()', 'HUB', 'VAD'))

                        if command.parsed['__name__'] == "flushed_out":
                            # process the outstanding DA if necessary
                            if outstanding_nlg_da:
                                nlg_commands.send(DMDA(outstanding_nlg_da, 'HUB', 'NLG'))
                                outstanding_nlg_da = None


                if vad_commands.poll():
                    command = vad_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "speech_start":
                            u_voice_activity = True

                            # interrupt the talking system
                            # this will be replaced with pausing the system when the necessary extension of pjsip
                            # is implemented
                            if s_voice_activity and s_last_voice_activity_time + 0.02 < current_time:
                                # if the system is still talking then flush the output
                                self.cfg['Logging']['session_logger'].barge_in("system")

                                # when a user barge in into the output, all the output pipe line
                                # must be flushed
                                nlg_commands.send(Command('flush()', 'HUB', 'NLG'))
                                s_voice_activity = False
                                s_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "speech_end":
                            u_voice_activity = False
                            u_last_voice_activity_time = time.time()

                        if command.parsed['__name__'] == "flushed":
                            # flush asr, when flushed, slu will be flushed
                            asr_commands.send(Command('flush()', 'HUB', 'ASR'))

                if asr_commands.poll():
                    command = asr_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush slu, when flushed, dm will be flushed
                            slu_commands.send(Command('flush()', 'HUB', 'SLU'))

                if slu_commands.poll():
                    command = slu_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush dm, when flushed, nlg will be flushed
                            dm_commands.send(Command('flush()', 'HUB', 'DM'))
                            dm_commands.send(Command('end_dialogue()', 'HUB', 'DM'))

                if dm_commands.poll():
                    command = dm_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "hangup":
                            # prepare for ending the call
                            hangup = True
                            self.cfg['Analytics'].track_event('vhub', 'system_hangup')

                        if command.parsed['__name__'] == "flushed":
                            # flush nlg, when flushed, tts will be flushed
                            nlg_commands.send(Command('flush()', 'HUB', 'NLG'))

                    elif isinstance(command, DMDA):
                        # record the time of the last system generated dialogue act
                        s_last_dm_activity_time = time.time()
                        number_of_turns += 1

                        if command.da != "silence()":
                            # if the DM generated non-silence dialogue act, then continue in processing it

                            if s_voice_activity and s_last_voice_activity_time + 0.02 < current_time:
                                # if the system is still talking then flush the output
                                self.cfg['Logging']['session_logger'].barge_in("system")

                                # when a user barge in into the output, all the output pipe line
                                # must be flushed
                                nlg_commands.send(Command('flush()', 'HUB', 'NLG'))
                                s_voice_activity = False
                                s_last_voice_activity_time = time.time()

                                # the DA will be send when all the following components are flushed
                                outstanding_nlg_da = command.da

                            else:
                                nlg_commands.send(DMDA(command.da, "HUB", "NLG"))

                if nlg_commands.poll():
                    command = nlg_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush tts, when flushed, vio will be flushed
                            tts_commands.send(Command('flush()', 'HUB', 'TTS'))

                if tts_commands.poll():
                    command = tts_commands.recv()
                    self.cfg['Logging']['system_logger'].info(command)

                    if isinstance(command, Command):
                        if command.parsed['__name__'] == "flushed":
                            # flush vio_out
                            vio_commands.send(Command('flush_out()', 'HUB', 'VIO'))

                current_time = time.time()

                s_diff = current_time - s_last_voice_activity_time
                u_diff = current_time - u_last_voice_activity_time

                if call_connected and \
                    not s_voice_activity and not u_voice_activity and \
                    s_diff > self.cfg['DM']['input_timeout'] and \
                    u_diff > self.cfg['DM']['input_timeout'] and \
                    current_time - u_last_input_timeout > self.cfg['DM']['input_timeout']:

                    u_last_input_timeout = time.time()
                    dm_commands.send(Command('timeout(silence_time="%0.3f")' % min(s_diff, u_diff), 'HUB', 'DM'))

                if hangup and s_last_dm_activity_time + 2.0 < current_time and \
                    s_voice_activity == False and s_last_voice_activity_time + 2.0 < current_time:
                    # we are ready to hangup only when all voice activity is finished
                    hangup = False
                    vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))

                if number_of_turns != -1 and current_time - call_start > self.cfg['VoipHub']['hard_time_limit'] or \
                    number_of_turns > self.cfg['VoipHub']['hard_turn_limit']:
                    # hard hangup due to the hard limits
                    call_start = 0
                    number_of_turns = -1
                    vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))

                if self.ncalls != 0 and not call_connected and s_last_dm_activity_time + 5.0 < current_time and ncalls >= self.ncalls:
                    break
                    
            # stop processes
            vio_commands.send(Command('stop()', 'HUB', 'VoipIO'))
            vad_commands.send(Command('stop()', 'HUB', 'VAD'))
            asr_commands.send(Command('stop()', 'HUB', 'ASR'))
            slu_commands.send(Command('stop()', 'HUB', 'SLU'))
            dm_commands.send(Command('stop()', 'HUB', 'DM'))
            nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
            tts_commands.send(Command('stop()', 'HUB', 'TTS'))

            # clean connections
            for c in command_connections:
                while c.poll():
                    c.recv()

            for c in non_command_connections:
                while c.poll():
                    c.recv()

            # wait for processes to stop
            # do not join, because in case of exception the join will not be successful
            # vio.join()
            # vad.join()
            # asr.join()
            # slu.join()
            # dm.join()
            # nlg.join()
            # tts.join()
            #cfg['Logging']['session_logger'].join()

        except KeyboardInterrupt:
            print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process().name
            self.close_event.set()
            return
        except:
            self.cfg['Logging']['system_logger'].exception('Uncaught exception in VHUB process.')
            self.close_event.set()
            raise

        print 'Exiting: %s. Setting close event' % multiprocessing.current_process().name
        self.close_event.set()
Example #10
0
    vad_audio_out, vad_child_audio_out = multiprocessing.Pipe() # used to read output audio from VAD

    tts_commands, tts_child_commands = multiprocessing.Pipe()   # used to send commands to TTS
    tts_text_in, tts_child_text_in = multiprocessing.Pipe()     # used to send TTS text

    command_connections = [vio_commands, vad_commands, tts_commands]

    non_command_connections = [vio_record, vio_child_record,
                               vio_play, vio_child_play,
                               vad_audio_out, vad_child_audio_out,
                               tts_text_in, tts_child_text_in]

    close_event = multiprocessing.Event()

    vio = VoipIO(cfg, vio_child_commands, vio_child_record, vio_child_play, close_event)
    vad = VAD(cfg, vad_child_commands, vio_record, vad_child_audio_out, close_event)
    tts = TTS(cfg, tts_child_commands, tts_child_text_in, vio_play, close_event)

    vio.start()
    vad.start()
    tts.start()

    cfg['Logging']['session_logger'].set_close_event(close_event)
    cfg['Logging']['session_logger'].set_cfg(cfg)
    cfg['Logging']['session_logger'].start()

    # init the system
    call_connected = False
    call_start = 0
    count_intro = 0
    intro_played = False
Example #11
0
File: webhub.py Project: AoJ/alex
    def run(self):
        # AIO pipes
        # used to send commands to VoipIO
        aio_commands, aio_child_commands = multiprocessing.Pipe()
        # I read from this connection recorded audio
        aio_record, aio_child_record = multiprocessing.Pipe()
        # I write in audio to be played
        aio_play, aio_child_play = multiprocessing.Pipe()

        # VAD pipes
        # used to send commands to VAD
        vad_commands, vad_child_commands = multiprocessing.Pipe()
        # used to read output audio from VAD
        vad_audio_out, vad_child_audio_out = multiprocessing.Pipe()

        # ASR pipes
        # used to send commands to ASR
        asr_commands, asr_child_commands = multiprocessing.Pipe()
        # used to read ASR hypotheses
        asr_hypotheses_out, asr_child_hypotheses = multiprocessing.Pipe()

        # SLU pipes
        # used to send commands to SLU
        slu_commands, slu_child_commands = multiprocessing.Pipe()
        # used to read SLU hypotheses
        slu_hypotheses_out, slu_child_hypotheses = multiprocessing.Pipe()

        # DM pipes
        # used to send commands to DM
        dm_commands, dm_child_commands = multiprocessing.Pipe()
        # used to read DM actions
        dm_actions_out, dm_child_actions = multiprocessing.Pipe()

        # NLG pipes
        # used to send commands to NLG
        nlg_commands, nlg_child_commands = multiprocessing.Pipe()
        # used to read NLG output
        nlg_text_out, nlg_child_text = multiprocessing.Pipe()

        # TTS pipes
        # used to send commands to TTS
        tts_commands, tts_child_commands = multiprocessing.Pipe()

        command_connections = [aio_commands, vad_commands, asr_commands,
                               slu_commands, dm_commands, nlg_commands,
                               tts_commands]

        non_command_connections = [aio_record, aio_child_record,
                                   aio_play, aio_child_play,
                                   vad_audio_out, vad_child_audio_out,
                                   asr_hypotheses_out, asr_child_hypotheses,
                                   slu_hypotheses_out, slu_child_hypotheses,
                                   dm_actions_out, dm_child_actions,
                                   nlg_text_out, nlg_child_text]

        # create the hub components
        close_event = multiprocessing.Event()
        aio = WebIO(self.cfg, aio_child_commands, aio_child_record, aio_child_play, close_event)
        vad = VAD(self.cfg, vad_child_commands, aio_record, vad_child_audio_out, close_event)
        asr = ASR(self.cfg, asr_child_commands, vad_audio_out, asr_child_hypotheses, close_event)
        slu = SLU(
            self.cfg, slu_child_commands, asr_hypotheses_out, slu_child_hypotheses, close_event)
        dm = DM(self.cfg, dm_child_commands, slu_hypotheses_out, dm_child_actions, close_event)
        nlg = NLG(self.cfg, nlg_child_commands, dm_actions_out, nlg_child_text, close_event)
        tts = TTS(self.cfg, tts_child_commands, nlg_text_out, aio_play, close_event)

        # start the hub components
        aio.start()
        vad.start()
        asr.start()
        slu.start()
        dm.start()
        nlg.start()
        tts.start()

        # init the system
        call_back_time = -1
        call_back_uri = None
        call_start = time.time()

        self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
        self.cfg['Logging']['system_logger'].session_system_log('config = ' + str(self.cfg))

        self.cfg['Logging']['session_logger'].session_start(
            self.cfg['Logging']['system_logger'].get_session_dir_name())
        self.cfg['Logging']['session_logger'].config('config = ' + str(self.cfg))
        self.cfg['Logging']['session_logger'].header(
            self.cfg['Logging']["system_name"], self.cfg['Logging']["version"])
        self.cfg['Logging']['session_logger'].input_source("aio")

        while 1:
            time.sleep(self.cfg['Hub']['main_loop_sleep_time'])

            if call_back_time != -1 and call_back_time < time.time():
                aio_commands.send(Command('make_call(destination="%s")' %
                                          call_back_uri, 'HUB', 'AIO'))
                call_back_time = -1
                call_back_uri = None

            if vad_commands.poll():
                command = vad_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "speech_start":
                        pass
                    if command.parsed['__name__'] == "speech_end":
                        pass

            if asr_commands.poll():
                command = asr_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

            if slu_commands.poll():
                command = slu_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

            if dm_commands.poll():
                command = dm_commands.recv()
                self.cfg['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "hangup":
                        # prepare for ending the call
                        pass

                    if command.parsed['__name__'] == "dm_da_generated":
                        # record the time of the last system generated dialogue act
                        pass

            if nlg_commands.poll():
                command = nlg_commands.recv()
                # TODO HACK
                # self.cfg['Logging']['system_logger'].info(command)

            if tts_commands.poll():
                command = tts_commands.recv()
                # TODO HACK
                # self.cfg['Logging']['system_logger'].info(command)

            current_time = time.time()

        # stop processes
        aio_commands.send(Command('stop()', 'HUB', 'AIO'))
        vad_commands.send(Command('stop()', 'HUB', 'VAD'))
        asr_commands.send(Command('stop()', 'HUB', 'ASR'))
        slu_commands.send(Command('stop()', 'HUB', 'SLU'))
        dm_commands.send(Command('stop()', 'HUB', 'DM'))
        nlg_commands.send(Command('stop()', 'HUB', 'NLG'))
        tts_commands.send(Command('stop()', 'HUB', 'TTS'))

        # clean connections
        for c in command_connections:
            while c.poll():
                c.recv()

        for c in non_command_connections:
            while c.poll():
                c.recv()

        # wait for processes to stop
        aio.join()
        vad.join()
        tts.join()
Example #12
0
def run(cfg, max_ncalls):
    try:
        cfg['Logging']['system_logger'].info(
            "Repeat After Me dialogue system\n" + "=" * 120)

        sample_sentences = load_sentences(
            cfg['RepeatAfterMe']['sentences_file'])

        vio_commands, vio_child_commands = multiprocessing.Pipe(
        )  # used to send commands to VoipIO
        vio_record, vio_child_record = multiprocessing.Pipe(
        )  # I read from this connection recorded audio
        vio_play, vio_child_play = multiprocessing.Pipe(
        )  # I write in audio to be played

        vad_commands, vad_child_commands = multiprocessing.Pipe(
        )  # used to send commands to VAD
        vad_audio_out, vad_child_audio_out = multiprocessing.Pipe(
        )  # used to read output audio from VAD

        tts_commands, tts_child_commands = multiprocessing.Pipe(
        )  # used to send commands to TTS
        tts_text_in, tts_child_text_in = multiprocessing.Pipe(
        )  # used to send TTS text

        command_connections = [vio_commands, vad_commands, tts_commands]

        non_command_connections = [
            vio_record, vio_child_record, vio_play, vio_child_play,
            vad_audio_out, vad_child_audio_out, tts_text_in, tts_child_text_in
        ]

        close_event = multiprocessing.Event()

        vio = VoipIO(cfg, vio_child_commands, vio_child_record, vio_child_play,
                     close_event)
        vad = VAD(cfg, vad_child_commands, vio_record, vad_child_audio_out,
                  close_event)
        tts = TTS(cfg, tts_child_commands, tts_child_text_in, vio_play,
                  close_event)

        vio.start()
        vad.start()
        tts.start()

        cfg['Logging']['session_logger'].set_close_event(close_event)
        cfg['Logging']['session_logger'].set_cfg(cfg)
        cfg['Logging']['session_logger'].start()

        # init the system
        call_connected = False
        call_start = 0
        count_intro = 0
        intro_played = False
        reject_played = False
        intro_id = 0
        last_intro_id = -1
        end_played = False
        s_voice_activity = False
        s_last_voice_activity_time = 0
        u_voice_activity = False
        u_last_voice_activity_time = 0
        ncalls = 0

        db = load_database(cfg['RepeatAfterMe']['call_db'])

        for remote_uri in db['calls_from_start_end_length']:
            num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(
                db, remote_uri)

            m = []
            m.append('')
            m.append('=' * 120)
            m.append('Remote SIP URI: %s' % remote_uri)
            m.append('-' * 120)
            m.append('Total calls:             %d' % num_all_calls)
            m.append('Total time (s):          %f' % total_time)
            m.append('Last 24h total calls:    %d' % last24_num_calls)
            m.append('Last 24h total time (s): %f' % last24_total_time)
            m.append('-' * 120)

            current_time = time.time()
            if last24_num_calls > cfg['RepeatAfterMe']['last24_max_num_calls'] or \
                    last24_total_time > cfg['RepeatAfterMe']['last24_max_total_time']:

                # add the remote uri to the black list
                vio_commands.send(
                    Command(
                        'black_list(remote_uri="%s",expire="%d")' %
                        (remote_uri,
                         current_time + cfg['RepeatAfterMe']['blacklist_for']),
                        'HUB', 'VoipIO'))
                m.append('BLACKLISTED')
            else:
                m.append('OK')

            m.append('-' * 120)
            m.append('')
            cfg['Logging']['system_logger'].info('\n'.join(m))

        call_back_time = -1
        call_back_uri = None

        while 1:
            # Check the close event.
            if close_event.is_set():
                print 'Received close event in: %s' % multiprocessing.current_process(
                ).name
                return

            time.sleep(cfg['Hub']['main_loop_sleep_time'])

            if vad_audio_out.poll():
                data_vad = vad_audio_out.recv()

            if call_back_time != -1 and call_back_time < time.time():
                vio_commands.send(
                    Command('make_call(destination="%s")' % call_back_uri,
                            'HUB', 'VoipIO'))
                call_back_time = -1
                call_back_uri = None

            # read all messages
            if vio_commands.poll():
                command = vio_commands.recv()

                if isinstance(command, Command):
                    if command.parsed[
                            '__name__'] == "incoming_call" or command.parsed[
                                '__name__'] == "make_call":
                        cfg['Logging']['system_logger'].session_start(
                            command.parsed['remote_uri'])
                        cfg['Logging']['session_logger'].session_start(
                            cfg['Logging']
                            ['system_logger'].get_session_dir_name())

                        cfg['Logging']['system_logger'].session_system_log(
                            'config = ' + unicode(cfg))
                        cfg['Logging']['system_logger'].info(command)

                        cfg['Logging']['session_logger'].config('config = ' +
                                                                unicode(cfg))
                        cfg['Logging']['session_logger'].header(
                            cfg['Logging']["system_name"],
                            cfg['Logging']["version"])
                        cfg['Logging']['session_logger'].input_source("voip")

                    if command.parsed['__name__'] == "rejected_call":
                        cfg['Logging']['system_logger'].info(command)

                        call_back_time = time.time() + cfg['RepeatAfterMe'][
                            'wait_time_before_calling_back']
                        # call back a default uri, if not defined call back the caller
                        if ('call_back_uri_subs' in cfg['RepeatAfterMe']
                            ) and cfg['RepeatAfterMe']['call_back_uri_subs']:
                            ru = command.parsed['remote_uri']
                            for pat, repl in cfg['RepeatAfterMe'][
                                    'call_back_uri_subs']:
                                ru = re.sub(pat, repl, ru)
                            call_back_uri = ru
                        elif ('call_back_uri' in cfg['RepeatAfterMe']
                              ) and cfg['RepeatAfterMe']['call_back_uri']:
                            call_back_uri = cfg['RepeatAfterMe'][
                                'call_back_uri']
                        else:
                            call_back_uri = command.parsed['remote_uri']

                    if command.parsed[
                            '__name__'] == "rejected_call_from_blacklisted_uri":
                        cfg['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']

                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(
                            db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append(
                            'Rejected incoming call from blacklisted URI: %s' %
                            remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' %
                                 last24_num_calls)
                        m.append('Last 24h total time (s): %f' %
                                 last24_total_time)
                        m.append('=' * 120)
                        m.append('')
                        cfg['Logging']['system_logger'].info('\n'.join(m))

                    if command.parsed['__name__'] == "call_connecting":
                        cfg['Logging']['system_logger'].info(command)

                    if command.parsed['__name__'] == "call_confirmed":
                        cfg['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']
                        num_all_calls, total_time, last24_num_calls, last24_total_time = get_stats(
                            db, remote_uri)

                        m = []
                        m.append('')
                        m.append('=' * 120)
                        m.append('Incoming call from :     %s' % remote_uri)
                        m.append('-' * 120)
                        m.append('Total calls:             %d' % num_all_calls)
                        m.append('Total time (s):          %f' % total_time)
                        m.append('Last 24h total calls:    %d' %
                                 last24_num_calls)
                        m.append('Last 24h total time (s): %f' %
                                 last24_total_time)
                        m.append('-' * 120)

                        if last24_num_calls > cfg['RepeatAfterMe']['last24_max_num_calls'] or \
                                last24_total_time > cfg['RepeatAfterMe']['last24_max_total_time']:

                            cfg['Logging']['session_logger'].turn("system")
                            tts_commands.send(
                                Command(
                                    'synthesize(text="%s",log="true")' %
                                    cfg['RepeatAfterMe']['rejected'], 'HUB',
                                    'TTS'))
                            call_connected = True
                            reject_played = True
                            s_voice_activity = True
                            vio_commands.send(
                                Command(
                                    'black_list(remote_uri="%s",expire="%d")' %
                                    (remote_uri, time.time() +
                                     cfg['RepeatAfterMe']['blacklist_for']),
                                    'HUB', 'VoipIO'))
                            m.append('CALL REJECTED')
                        else:
                            # init the system
                            call_connected = True
                            call_start = time.time()
                            count_intro = 0
                            intro_played = False
                            reject_played = False
                            end_played = False
                            s_voice_activity = False
                            s_last_voice_activity_time = 0
                            u_voice_activity = False
                            u_last_voice_activity_time = 0

                            intro_id, last_intro_id = play_intro(
                                cfg, tts_commands, intro_id, last_intro_id)

                            m.append('CALL ACCEPTED')

                        m.append('=' * 120)
                        m.append('')
                        cfg['Logging']['system_logger'].info('\n'.join(m))

                        try:
                            db['calls_from_start_end_length'][
                                remote_uri].append([time.time(), 0, 0])
                        except:
                            db['calls_from_start_end_length'][remote_uri] = [
                                [time.time(), 0, 0],
                            ]
                        save_database(cfg['RepeatAfterMe']['call_db'], db)

                    if command.parsed['__name__'] == "call_disconnected":
                        cfg['Logging']['system_logger'].info(command)

                        remote_uri = command.parsed['remote_uri']

                        vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                        vad_commands.send(Command('flush()', 'HUB', 'VAD'))
                        tts_commands.send(Command('flush()', 'HUB', 'TTS'))

                        cfg['Logging']['system_logger'].session_end()
                        cfg['Logging']['session_logger'].session_end()

                        try:
                            s, e, l = db['calls_from_start_end_length'][
                                remote_uri][-1]

                            if e == 0 and l == 0:
                                # there is a record about last confirmed but not disconnected call
                                db['calls_from_start_end_length'][remote_uri][
                                    -1] = [s, time.time(),
                                           time.time() - s]
                                save_database('call_db.pckl', db)
                        except KeyError:
                            # disconnecting call which was not confirmed for URI calling for the first time
                            pass

                        intro_played = False
                        call_connected = False
                        ncalls += 1

                    if command.parsed['__name__'] == "play_utterance_start":
                        cfg['Logging']['system_logger'].info(command)
                        s_voice_activity = True

                    if command.parsed['__name__'] == "play_utterance_end":
                        cfg['Logging']['system_logger'].info(command)

                        s_voice_activity = False
                        s_last_voice_activity_time = time.time()

                        if command.parsed['user_id'] == last_intro_id:
                            intro_played = True
                            s_last_voice_activity_time = 0

            if vad_commands.poll():
                command = vad_commands.recv()
                cfg['Logging']['system_logger'].info(command)

                if isinstance(command, Command):
                    if command.parsed['__name__'] == "speech_start":
                        u_voice_activity = True
                    if command.parsed['__name__'] == "speech_end":
                        u_voice_activity = False
                        u_last_voice_activity_time = time.time()

            if tts_commands.poll():
                command = tts_commands.recv()
                cfg['Logging']['system_logger'].info(command)

            current_time = time.time()

            #  print
            #  print intro_played, end_played
            #  print s_voice_activity, u_voice_activity,
            #  print call_start,  current_time, u_last_voice_activity_time, s_last_voice_activity_time
            #  print current_time - s_last_voice_activity_time > 5, u_last_voice_activity_time - s_last_voice_activity_time > 0

            if reject_played == True and s_voice_activity == False:
                # be careful it does not hangup immediately
                reject_played = False
                vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))
                vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                vad_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                tts_commands.send(Command('flush()', 'HUB', 'VoipIO'))

            if intro_played and current_time - call_start > cfg[
                    'RepeatAfterMe'][
                        'max_call_length'] and s_voice_activity == False:
                # too long call
                if not end_played:
                    s_voice_activity = True
                    last_intro_id = str(intro_id)
                    intro_id += 1
                    cfg['Logging']['session_logger'].turn("system")
                    tts_commands.send(
                        Command(
                            'synthesize(text="%s",log="true")' %
                            cfg['RepeatAfterMe']['closing'], 'HUB', 'TTS'))
                    end_played = True
                else:
                    intro_played = False
                    # be careful it does not hangup immediately
                    vio_commands.send(Command('hangup()', 'HUB', 'VoipIO'))
                    vio_commands.send(Command('flush()', 'HUB', 'VoipIO'))
                    vad_commands.send(Command('flush()', 'HUB', 'VAD'))
                    tts_commands.send(Command('flush()', 'HUB', 'TTS'))

            if intro_played and \
                s_voice_activity == False and \
                u_voice_activity == False and \
                current_time - s_last_voice_activity_time > 5 and current_time - u_last_voice_activity_time > 0.6:

                if 'silence' not in cfg['RepeatAfterMe'] or not cfg[
                        'RepeatAfterMe']['silence']:
                    s_voice_activity = True

                    s1 = ram()
                    cfg['Logging']['session_logger'].turn("system")
                    tts_commands.send(
                        Command('synthesize(text="%s",log="true")' % s1, 'HUB',
                                'TTS'))
                    s2 = sample_sentence(sample_sentences)
                    tts_commands.send(
                        Command('synthesize(text="%s",log="true")' % s2, 'HUB',
                                'TTS'))

                    s = s1 + ' ' + s2

                    m = []
                    m.append('=' * 120)
                    m.append('Say: ' + s)
                    m.append('=' * 120)

                    cfg['Logging']['system_logger'].info('\n'.join(m))

            if ncalls != 0 and not call_connected and s_last_voice_activity_time + 5.0 < current_time and ncalls >= max_ncalls:
                break

        # stop processes
        vio_commands.send(Command('stop()', 'HUB', 'VoipIO'))
        vad_commands.send(Command('stop()', 'HUB', 'VAD'))
        tts_commands.send(Command('stop()', 'HUB', 'TTS'))

        # clean connections
        for c in command_connections:
            while c.poll():
                c.recv()

        for c in non_command_connections:
            while c.poll():
                c.recv()

        # wait for processes to stop
        # do not join, because in case of exception the join will not be successful
        #vio.join()
        #system_logger.debug('VIO stopped.')
        #vad.join()
        #system_logger.debug('VAD stopped.')
        #tts.join()
        #system_logger.debug('TTS stopped.')

    except KeyboardInterrupt:
        print 'KeyboardInterrupt exception in: %s' % multiprocessing.current_process(
        ).name
        close_event.set()
        return
    except:
        cfg['Logging']['system_logger'].exception(
            'Uncaught exception in SW_HUB process.')
        close_event.set()
        raise

    print 'RAM HUB: Exiting: %s. Setting close event' % multiprocessing.current_process(
    ).name
    close_event.set()