def _run_story_test(self, sender_id, story):
utils.print_color('\n## ' + story['title'].upper(),
utils.bcolors.OKBLUE)
self.failed = False
logger.debug('Starting story: {story}'.format(story=story))
for index, step in enumerate(story['steps']):
if self.failed is True:
return
utterance = step.pop(0)
logger.debug('Starting step {index}: {utterance}'.format(
index=index, utterance=utterance))
messages = self._get_response(sender_id, utterance)
logger.debug('messages = {}'.format(messages))
logger.debug('expected = {}'.format(step))
utils.print_color('* {}'.format(utterance), utils.bcolors.OKBLUE)
for message in messages:
expected = step.pop(0) if len(step) > 0 else None
if message == expected:
utils.print_color(" - {}".format(message),
utils.bcolors.OKGREEN)
else:
utils.print_color(
" - {} (exp: {})".format(message, expected),
utils.bcolors.FAIL)
utils.print_color(
"TEST CASE INTERRUPTED)".format(message, expected),
utils.bcolors.FAIL)
self.failed = True
def run_story_test(agent, domain, input_channel, sender_id, story):
global failed
utils.print_color('\n## ' + story['title'].upper(), utils.bcolors.OKBLUE)
failed = False
for index, step in enumerate(story['steps']):
if failed is True:
return failed
utterance = step.pop(0)
def on_response(response, recipient_id, proc):
global failed
if len(step) > 0:
expected = step.pop(0)
if response == expected:
utils.print_color(" - {}".format(response), utils.bcolors.OKGREEN)
else:
utils.print_color(" - {} (exp: {})".format(response, expected), utils.bcolors.FAIL)
utils.print_color("TEST CASE INTERRUPTED)".format(response, expected), utils.bcolors.FAIL)
# _print_slot_values(proc, recipient_id)
_restart_tracker(proc, recipient_id)
failed = True
else:
utils.print_color(" - {} (not in case)".format(response), utils.bcolors.BOLD)
utils.print_color(utterance, utils.bcolors.OKGREEN)
input_channel.on_message(
UserMessage(utterance, TestOutputChannel(on_response, domain, agent.processor), sender_id))
return failed
def _record_messages(self, on_message, max_message_limit=None):
utils.print_color("Bot loaded. Type a message and press enter : ",
utils.bcolors.OKGREEN)
client = None
my_user_name = ''
self.client = SlackClient(self.token)
self.client.rtm_connect()
self.my_user_name = self.client.server.username
print("Connected to Slack.")
num_messages = 0
while max_message_limit is None or num_messages < max_message_limit:
try:
input = self.client.rtm_read()
if input:
for action in input:
if self.debug_mode:
print(action)
if 'type' in action and action['type'] == "message":
# Uncomment to only respond to messages addressed to us.
# if 'text' in action
# and action['text'].lower().startswith(self.my_user_name):
#print(action['text'])
text = action["text"]
channel = action["channel"]
#print("User said",text)
# self.process_message(action)
out_channel = SlackRTMOutputChannel(
self.client, channel)
on_message(UserMessage(text, out_channel))
num_messages += 1
else:
sleep(1)
except Exception as e:
print("Exception: ", e.message)
def record_messages(
endpoint, # type: EndpointConfig
sender_id=UserMessage.DEFAULT_SENDER_ID, # type: Text
max_message_limit=None, # type: Optional[int]
on_finish=None, # type: Optional[Callable[[], None]]
finetune=False # type: bool
):
"""Read messages from the command line and print bot responses."""
try:
exit_text = INTENT_MESSAGE_PREFIX + 'stop'
utils.print_color(
"Bot loaded. Type a message and press enter "
"(use '{}' to exit). ".format(exit_text), utils.bcolors.OKGREEN)
try:
domain = retrieve_domain(endpoint)
except requests.exceptions.ConnectionError:
logger.exception("Failed to connect to rasa core server at '{}'. "
"Is the server running?".format(endpoint.url))
return
intents = [next(iter(i)) for i in (domain.get("intents") or [])]
num_messages = 0
while not utils.is_limit_reached(num_messages, max_message_limit):
try:
if is_listening_for_message(sender_id, endpoint):
_enter_user_message(sender_id, endpoint, exit_text)
_validate_nlu(intents, endpoint, sender_id)
_predict_till_next_listen(endpoint,
sender_id,
finetune=finetune)
num_messages += 1
except RestartConversation:
send_event(endpoint, sender_id, {"event": "restart"})
send_event(endpoint, sender_id, {
"event": "action",
"name": ACTION_LISTEN_NAME
})
logger.info("Restarted conversation, starting a new one.")
except UndoLastStep:
_undo_latest(sender_id, endpoint)
_print_history(sender_id, endpoint)
except Exception:
logger.exception("An exception occurred while recording messages.")
raise
finally:
if on_finish:
on_finish()
def _print_help(skip_visualization: bool) -> None:
"""Print some initial help message for the user."""
if not skip_visualization:
visualization_help = "Visualisation at {}/visualization.html." \
"".format(DEFAULT_SERVER_URL)
else:
visualization_help = ""
utils.print_color("Bot loaded. {}\n"
"Type a message and press enter "
"(press 'Ctr-c' to exit). "
"".format(visualization_help), utils.bcolors.OKGREEN)
def _record_messages(self, on_message, max_message_limit=None):
utils.print_color("Bot loaded. Type a message and press enter: ",
utils.bcolors.OKGREEN)
num_messages = 0
while max_message_limit is None or num_messages < max_message_limit:
text = input().strip()
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
if text == INTENT_MESSAGE_PREFIX + 'stop':
return
on_message(UserMessage(text, ConsoleOutputChannel(),
self.sender_id))
num_messages += 1
def process_failed_story(failed_story):
print('\n')
for line in failed_story.splitlines():
is_intent = is_an_intent_line(line)
if re.search('<!--.*-->', line):
error_prediction = re.search('<!--(.*)-->', line) \
.group(1).strip().split(':')
error_message = ''
if is_intent:
error_message += ' Intent failed: Predicted intent' + \
('{} for message \'{}\''.
format(error_prediction[1],
error_prediction[2].strip())
if len(error_prediction) == 3
else error_prediction[1])
else:
error_message += ' Utter failed: Predicted utter' \
+ error_prediction[1]
line = re.sub('<!--(.*)-->', '', line)
utils.print_color("-" * len(error_message), FAILED_COLOR)
utils.print_color(line, FAILED_COLOR)
utils.print_color(error_message, FAILED_COLOR)
utils.print_color("-" * len(error_message), FAILED_COLOR)
else:
print(line)
def _record_messages(self, on_message, max_message_limit=None):
utils.print_color("Bot loaded. Type a message and press enter: ",
utils.bcolors.OKGREEN)
num_messages = 0
while max_message_limit is None or num_messages < max_message_limit:
text = input().strip()
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
if text == INTENT_MESSAGE_PREFIX + 'stop':
return
on_message(UserMessage(text, ConsoleOutputChannel(),
self.sender_id))
num_messages += 1
def process_failed_story(failed_story):
print("\n")
for line in failed_story.splitlines():
is_intent = is_an_intent_line(line)
if re.search("<!--.*-->", line):
error_prediction = (re.search("<!--(.*)-->",
line).group(1).strip().split(":"))
error_message = ""
if is_intent:
error_message += " Intent failed: Predicted intent" + (
"{} for message '{}'".format(error_prediction[1],
error_prediction[2].strip())
if len(error_prediction) == 3 else error_prediction[1])
else:
error_message += (" Utter failed: Predicted utter" +
error_prediction[1])
line = re.sub("<!--(.*)-->", "", line)
utils.print_color("-" * len(error_message), FAILED_COLOR)
utils.print_color(line, FAILED_COLOR)
utils.print_color(error_message, FAILED_COLOR)
utils.print_color("-" * len(error_message), FAILED_COLOR)
else:
print(line)
def _record_messages(self, on_message, max_message_limit=None):
utils.print_color("Bot loaded. Type a message and press enter : ",
utils.bcolors.OKGREEN)
num_messages = 0
while max_message_limit is None or num_messages < max_message_limit:
text = input().strip()
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
if text == '_stop':
import os
# sys.exit(1)
os._exit(1)
on_message(UserMessage(text, ConsoleOutputChannel()))
num_messages += 1
def record_messages(endpoint,
sender_id=UserMessage.DEFAULT_SENDER_ID,
max_message_limit=None,
on_finish=None,
get_next_message=None):
"""Read messages from the command line and print bot responses."""
try:
exit_text = INTENT_MESSAGE_PREFIX + 'stop'
if not get_next_message:
# default to reading messages from the command line
get_next_message = console.get_cmd_input
utils.print_color("Bot loaded. Type a message and press enter "
"(use '{}' to exit). ".format(exit_text),
utils.bcolors.OKGREEN)
try:
domain = retrieve_domain(endpoint)
except requests.exceptions.ConnectionError:
logger.exception("Failed to connect to rasa core server at '{}'. "
"Is the server running?".format(endpoint.url))
return
intents = [next(iter(i)) for i in (domain.get("intents") or [])]
num_messages = 0
finished = False
while (not finished and
not utils.is_limit_reached(num_messages, max_message_limit)):
print("Next user input:")
text = get_next_message()
if text == exit_text:
break
send_message(endpoint, sender_id, text)
finished = predict_till_next_listen(endpoint,
intents,
sender_id)
num_messages += 1
except Exception:
logger.exception("An exception occurred while recording messages.")
raise
finally:
if on_finish:
on_finish()
def _record_messages(self, on_message, max_message_limit=None):
utils.print_color(textwrap.dedent("""
Bot loaded with voice capabilities!
Note that in this mode the bot will do the following things:
- When the answer it's too long (more than 250 characters) the bot will
just say that it found an answer and then print the real answer to the
console this is done because 'speaking' a long answer can take a lot of time
- When multiple answers to a question are found then the bot will only say
that it found multiple answers and it will print each answer to the console.
This is also done to save time (besides, listening to multiple answers can be confusing)
""") ,
utils.bcolors.OKGREEN)
num_messages = 0
while max_message_limit is None or num_messages < max_message_limit:
try:
if self.prefer_text_input: # if we want to do normal text input and have audio output:
text = input("\nYour Input: ").strip()
else: # if we want to use text input as well
with self.MICROPHONE as source:
self.SRECOGNIZER.adjust_for_ambient_noise(source)
input(
"\nBot is ready! Press enter when you're ready to speak.")
audio = self.SRECOGNIZER.listen(source)
print("Bot is recognizing what you said ...")
text = self.SRECOGNIZER.recognize_google(audio)
print("You said: " + text)
except Exception:
text = input(
"Voice does not work :( input your query here: ").strip()
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
if text == INTENT_MESSAGE_PREFIX + 'stop':
return
on_message(UserMessage(text, VoiceOutputChannel(),
self.sender_id))
num_messages += 1
def _record_messages(self, on_message, max_message_limit=None):
utils.print_color("Bot loaded. Type a message and press enter: ",
utils.bcolors.OKGREEN)
self.output_channel.say("Bot loaded, Say something")
num_messages = 0
while max_message_limit is None or num_messages < max_message_limit:
print("Getting new speech input")
text = self.input_channel.get_next_sentence()
while text is None:
self.output_channel.say("Didn't understand, please retry")
text = self.input_channel.get_next_sentence()
print("Undersood:" + text)
if six.PY2:
# in python 2 input doesn't return unicode values
text = text.decode("utf-8")
if text == INTENT_MESSAGE_PREFIX + 'stop':
return
on_message(UserMessage(text, SpeechOutputChannel(), self.sender_id))
num_messages += 1
def print_bot_output(message, color=utils.bcolors.OKBLUE):
if "text" in message:
utils.print_color(message.get("text"), color)
if "image" in message:
utils.print_color("Image: " + message.get("image"), color)
if "attachment" in message:
utils.print_color("Attachment: " + message.get("attachment"), color)
if "buttons" in message:
for idx, button in enumerate(message.get("buttons")):
button_str = "Buttons:\n" + button_to_string(button, idx)
utils.print_color(button_str, color)
if "elements" in message:
for idx, element in enumerate(message.get("elements")):
element_str = "Elements:\n" + element_to_string(element, idx)
utils.print_color(element_str, color)
def send_text_message(self, recipient_id, message):
# cmd output
utils.print_color(message, self.default_output_color)
# send response message via http
"""
url = self.url
print(f'Sending text message {message} to output address {url}')
data = {"sender": "bot", "message": message}
data_json = json.dumps(data)
headers = {"Content-Type": "application/json"}
requests.post(
url=url,
data=data_json,
headers=headers
)
"""
return self.tts.utter_voice_message(message)
def print_evaluation_statistics():
utils.print_color('EVALUATION RESULTS:', BOLD_COLOR)
if len(all_failed_stories) == 0:
utils.print_color(
'\n' + 'CONGRATULATIONS! ALL THE EVALUATED '
'STORIES HAVE PASSED!', PASSED_COLOR)
else:
utils.print_color('\n' + 'FAILED STORIES:', FAILED_COLOR)
for failed_story in all_failed_stories:
utils.print_color(failed_story, FAILED_COLOR)
sys.exit(1)
def print_evaluation_statistics():
utils.print_color("EVALUATION RESULTS:", BOLD_COLOR)
if len(all_failed_stories) == 0:
utils.print_color(
"\n" + "CONGRATULATIONS! ALL THE EVALUATED "
"STORIES HAVE PASSED!",
PASSED_COLOR,
)
else:
utils.print_color("\n" + "FAILED STORIES:", FAILED_COLOR)
for failed_story in all_failed_stories:
utils.print_color(failed_story, FAILED_COLOR)
sys.exit(1)
def print_bot_output(message, color=utils.bcolors.OKBLUE):
if "text" in message:
utils.print_color(message.get("text"), color)
if "image" in message:
utils.print_color("Image: " + message.get("image"), color)
if "attachment" in message:
utils.print_color("Attachment: " + message.get("attachment"), color)
if "buttons" in message:
for idx, button in enumerate(message.get("buttons")):
button_str = button_to_string(button, idx)
utils.print_color(button_str, color)
def record_messages(server_url=DEFAULT_SERVER_URL,
auth_token=None,
sender_id=UserMessage.DEFAULT_SENDER_ID,
max_message_limit=None,
use_response_stream=True,
on_finish=None):
"""Read messages from the command line and print bot responses."""
auth_token = auth_token if auth_token else ""
exit_text = INTENT_MESSAGE_PREFIX + 'stop'
utils.print_color("Bot loaded. Type a message and press enter "
"(use '{}' to exit): ".format(exit_text),
utils.bcolors.OKGREEN)
num_messages = 0
while not utils.is_limit_reached(num_messages, max_message_limit):
text = get_cmd_input()
if text == exit_text:
break
if use_response_stream:
bot_responses = send_message_receive_stream(server_url,
auth_token,
sender_id, text)
else:
bot_responses = send_message_receive_block(server_url,
auth_token,
sender_id, text)
for response in bot_responses:
print_bot_output(response)
num_messages += 1
if on_finish:
on_finish()
def record_messages(server_url=DEFAULT_SERVER_URL,
auth_token=None,
sender_id=UserMessage.DEFAULT_SENDER_ID,
max_message_limit=None,
use_response_stream=True,
on_finish=None):
"""Read messages from the command line and print bot responses."""
auth_token = auth_token if auth_token else ""
exit_text = INTENT_MESSAGE_PREFIX + 'stop'
utils.print_color("Bot loaded. Type a message and press enter "
"(use '{}' to exit): ".format(exit_text),
utils.bcolors.OKGREEN)
num_messages = 0
while not utils.is_limit_reached(num_messages, max_message_limit):
text = get_cmd_input()
if text == exit_text:
break
if use_response_stream:
bot_responses = send_message_receive_stream(server_url,
auth_token,
sender_id, text)
else:
bot_responses = send_message_receive_block(server_url,
auth_token,
sender_id, text)
for response in bot_responses:
print_bot_output(response)
num_messages += 1
if on_finish:
on_finish()
def on_response(response, recipient_id, proc):
global failed
if len(step) > 0:
expected = step.pop(0)
if response == expected:
utils.print_color(" - {}".format(response), utils.bcolors.OKGREEN)
else:
utils.print_color(" - {} (exp: {})".format(response, expected), utils.bcolors.FAIL)
utils.print_color("TEST CASE INTERRUPTED)".format(response, expected), utils.bcolors.FAIL)
# _print_slot_values(proc, recipient_id)
_restart_tracker(proc, recipient_id)
failed = True
else:
utils.print_color(" - {} (not in case)".format(response), utils.bcolors.BOLD)
def send_text_message(self, recipient_id, message):
# type: (Text, Text) -> None
utils.print_color(message, self.default_output_color)
def send_text_message(self, recipient_id, message):
# type: (Text, Text) -> None
utils.print_color(message, self.default_output_color)
def run_evaluation(file_to_evaluate,
fail_on_prediction_errors=False,
max_stories=None,
use_e2e=False):
_endpoints = AvailableEndpoints.read_endpoints(None)
_interpreter = NaturalLanguageInterpreter.create(NLU_DIR)
_agent = load_agent(CORE_DIR,
interpreter=_interpreter,
endpoints=_endpoints)
completed_trackers = _generate_trackers(file_to_evaluate, _agent,
max_stories, use_e2e)
story_evaluation, _ = collect_story_predictions(completed_trackers, _agent,
fail_on_prediction_errors,
use_e2e)
_failed_stories = story_evaluation.failed_stories
_num_stories = len(completed_trackers)
_file_result = FileResult(num_stories=_num_stories,
num_failed_stories=len(_failed_stories))
file_message = "EVALUATING STORIES FOR FILE '{}':".format(file_to_evaluate)
utils.print_color('\n' + '#' * 80, BOLD_COLOR)
utils.print_color(file_message, BOLD_COLOR)
files_results[file_to_evaluate] = _file_result
if len(_failed_stories) == 0:
success_message = "The stories have passed for file '{}'!!" \
.format(file_to_evaluate)
utils.print_color('\n' + '=' * len(success_message), BLUE_COLOR)
utils.print_color(success_message, BLUE_COLOR)
utils.print_color('=' * len(success_message), BLUE_COLOR)
else:
for failed_story in _failed_stories:
process_failed_story(failed_story.export_stories())
story_name = re.search('## (.*)',
failed_story.export_stories()).group(1)
all_failed_stories.append(file_to_evaluate + ' - ' + story_name)
utils.print_color('#' * 80 + '\n', BOLD_COLOR)
def send_text_message(self, recipient_id, message):
post(channel="D83FA57AL", message=message)
# type: (Text, Text) -> None
utils.print_color(message, self.default_output_color)
