def test_legacy_protocol(self):
v3_hostname = "https://westus.api.cognitive.microsoft.com/qnamaker/v3.0"
v3_legacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v3_hostname)
legacy_qna = QnAMaker(v3_legacy_endpoint)
is_legacy = True
v4_hostname = "https://UpdatedNonLegacyQnaHostName.azurewebsites.net/qnamaker"
nonlegacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v4_hostname)
v4_qna = QnAMaker(nonlegacy_endpoint)
self.assertEqual(is_legacy, legacy_qna._is_legacy_protocol)
self.assertNotEqual(is_legacy, v4_qna._is_legacy_protocol)
def __init__(self, config: Config):
self.qna_maker = QnAMaker(
QnAMakerEndpoint(
knowledge_base_id=config["QNA_KNOWLEDGEBASE_ID"],
endpoint_key=config["QNA_ENDPOINT_KEY"],
host=config["QNA_ENDPOINT_HOST"],
))
def __init__(self, user_state: UserState,
luis_recognizer: InsuranceQueryRecognizer,
insurance_renewal_dialog: InsuranceRenewalDialog,
reservation_booking_dialog: ReservationBookingDialog):
super(MainDialog, self).__init__(MainDialog.__name__)
self.qna_maker = QnAMaker(
QnAMakerEndpoint(knowledge_base_id=config.QNA_KNOWLEDGEBASE_ID,
endpoint_key=config.QNA_ENDPOINT_KEY,
host=config.QNA_ENDPOINT_HOST))
self.user_profile_accessor = user_state.create_property("UserData")
self.insurance_renewal_dialog_id = insurance_renewal_dialog.id
self.reservation_booking_dialog_id = reservation_booking_dialog.id
self._luis_recognizer = luis_recognizer
self.user_state = user_state
self.add_dialog(TextPrompt(TextPrompt.__name__))
self.add_dialog(ChoicePrompt(INS_PROMPT_OPTIONS))
self.add_dialog(insurance_renewal_dialog)
self.add_dialog(reservation_booking_dialog)
self.add_dialog(
WaterfallDialog("WFDialog", [
self.intro_step, self.name_process_step, self.luis_query_step,
self.closing_step
]))
self.initial_dialog_id = "WFDialog"
def __init__(self,
config: DefaultConfig,
conversation_state: ConversationState,
user_state: UserState,
promptInput=False,
getClaim=False):
self.qna_maker = QnAMaker(
QnAMakerEndpoint(
knowledge_base_id=config.QNA_KNOWLEDGEBASE_ID,
endpoint_key=config.QNA_ENDPOINT_KEY,
host=config.QNA_ENDPOINT_HOST,
))
self.promptInput = promptInput
self.getClaim = getClaim
if conversation_state is None:
raise TypeError(
"[CustomPromptBot]: Missing parameter. conversation_state is required but None was given"
)
if user_state is None:
raise TypeError(
"[CustomPromptBot]: Missing parameter. user_state is required but None was given"
)
self.conversation_state = conversation_state
self.user_state = user_state
self.flow_accessor = self.conversation_state.create_property(
"ConversationFlow")
self.profile_accessor = self.user_state.create_property("UserProfile")
self.flow_accessor = self.conversation_state.create_property(
"ConversationFlow")
self.profile_accessor = self.user_state.create_property("UserProfile")
def __init__(self, user_state: UserState, config: Config):
self.qna_maker = QnAMaker(
QnAMakerEndpoint(
knowledge_base_id=config.QNA_KNOWLEDGEBASE_ID,
endpoint_key=config.QNA_ENDPOINT_KEY,
host=config.QNA_ENDPOINT_HOST,
))
if user_state is None:
raise TypeError(
"[WelcomeUserBot]: Missing parameter. user_state is required but None was given"
)
self._user_state = user_state
self.user_state_accessor = self._user_state.create_property(
"WelcomeUserState")
self.WELCOME_MESSAGE = """Type 'hello', 'hi','bye', 'docs' and 'intro' to explore more. Try it now, type 'hi'"""
self.INFO_MESSAGE = """Welcome to ADC CHAT BOT"""
self.LOCALE_MESSAGE = """" """
self.PATTERN_MESSAGE = """Type 'hello', 'hi','bye', 'docs' and 'intro' to explore more. Try it now, type 'hi'"""
def __init__(self, config: DefaultConfig):
self.qna_maker = QnAMaker(
QnAMakerEndpoint(
knowledge_base_id=config.QNA_KNOWLEDGEBASE_ID,
endpoint_key=config.QNA_ENDPOINT_KEY,
host=config.QNA_ENDPOINT_HOST,
))
def test_v2_legacy_endpoint(self):
v2_hostname = "https://westus.api.cognitive.microsoft.com/qnamaker/v2.0"
v2_legacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v2_hostname)
with self.assertRaises(ValueError):
QnAMaker(v2_legacy_endpoint)
def __init__(self, conversation_state: ConversationState,
user_state: UserState):
self.config_reader = ConfigReader()
self.configuration = self.config_reader.read_config()
self.luis_app_id = self.configuration['LUIS_APP_ID']
self.luis_endpoint_key = self.configuration['LUIS_ENDPOINT_KEY']
self.luis_endpoint = self.configuration['LUIS_ENDPOINT']
self.luis_app = LuisApplication(self.luis_app_id,
self.luis_endpoint_key,
self.luis_endpoint)
self.luis_options = LuisPredictionOptions(include_all_intents=True,
include_instance_data=True)
self.luis_recognizer = LuisRecognizer(
application=self.luis_app,
prediction_options=self.luis_options,
include_api_results=True)
self.qna_knowledge_base_id = self.configuration["QNA_KNOWLEDGEBASE_ID"]
self.qna_endpoint_key = self.configuration["QNA_ENDPOINT_KEY"]
self.qna_host = self.configuration["QNA_ENDPOINT_HOST"]
self.qna_maker = QnAMaker(
QnAMakerEndpoint(knowledge_base_id=self.qna_knowledge_base_id,
endpoint_key=self.qna_endpoint_key,
host=self.qna_host))
self.log = Log()
self.IntentIdentified = False
self.intent = 'none'
self.stat = 'init'
self.city = ""
self.score = ""
if conversation_state is None:
raise TypeError(
"[StateManagementBot]: Missing parameter. conversation_state is required but None was given"
)
if user_state is None:
raise TypeError(
"[StateManagementBot]: Missing parameter. user_state is required but None was given"
)
self.conversation_state = conversation_state
self.user_state = user_state
self.conversation_data_accessor = self.conversation_state.create_property(
"ConversationData")
self.user_profile_accessor = self.user_state.create_property(
"UserProfile")
def __init__(self, config: DefaultConfig):
self.qna_maker = QnAMaker(
QnAMakerEndpoint(
knowledge_base_id=config.QNA_KNOWLEDGEBASE_ID,
endpoint_key=config.QNA_ENDPOINT_KEY,
host=config.QNA_ENDPOINT_HOST,
))
# If the includeApiResults parameter is set to true, as shown below, the full response
# from the LUIS api will be made available in the properties of the RecognizerResult
luis_application = LuisApplication(
config.LUIS_APP_ID,
config.LUIS_API_KEY,
"https://" + config.LUIS_API_HOST_NAME,
)
luis_options = LuisPredictionOptions(include_all_intents=True,
include_instance_data=True)
self.recognizer = LuisRecognizer(luis_application, luis_options, True)
def __init__(self, config: Config):
self.qna_maker = QnAMaker(
QnAMakerEndpoint(
knowledge_base_id=config.QNA_KNOWLEDGEBASE_ID,
endpoint_key=config.QNA_ENDPOINT_KEY,
host=config.QNA_ENDPOINT_HOST,
), QnAMakerOptions(score_threshold=0.9))
luis_application = LuisApplication(
config.LUIS_APP_ID,
config.LUIS_API_KEY,
"https://" + config.LUIS_API_HOST_NAME,
)
luis_options = LuisPredictionOptions(include_all_intents=True,
include_instance_data=True)
self.recognizer = LuisRecognizer(luis_application, luis_options, True)
self.db_func = DB_function()
self.favor = my_favorite()
self.history = history()
def __init__(self):
qna_endpoint = QnAMakerEndpoint("", "", "")
self.qna_maker = QnAMaker(qna_endpoint)
def test_endpoint_with_empty_endpoint_key(self):
empty_endpoint_key = ""
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, empty_endpoint_key,
self._host)
class QnaApplicationTest(aiounittest.AsyncTestCase):
# Note this is NOT a real QnA Maker application ID nor a real QnA Maker subscription-key
# theses are GUIDs edited to look right to the parsing and validation code.
_knowledge_base_id: str = "f028d9k3-7g9z-11d3-d300-2b8x98227q8w"
_endpoint_key: str = "1k997n7w-207z-36p3-j2u1-09tas20ci6011"
_host: str = "https://dummyqnahost.azurewebsites.net/qnamaker"
tests_endpoint = QnAMakerEndpoint(_knowledge_base_id, _endpoint_key, _host)
def test_qnamaker_construction(self):
# Arrange
endpoint = self.tests_endpoint
# Act
qna = QnAMaker(endpoint)
endpoint = qna._endpoint
# Assert
self.assertEqual("f028d9k3-7g9z-11d3-d300-2b8x98227q8w",
endpoint.knowledge_base_id)
self.assertEqual("1k997n7w-207z-36p3-j2u1-09tas20ci6011",
endpoint.endpoint_key)
self.assertEqual("https://dummyqnahost.azurewebsites.net/qnamaker",
endpoint.host)
def test_endpoint_with_empty_kbid(self):
empty_kbid = ""
with self.assertRaises(TypeError):
QnAMakerEndpoint(empty_kbid, self._endpoint_key, self._host)
def test_endpoint_with_empty_endpoint_key(self):
empty_endpoint_key = ""
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, empty_endpoint_key,
self._host)
def test_endpoint_with_emptyhost(self):
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, self._endpoint_key, "")
def test_qnamaker_with_none_endpoint(self):
with self.assertRaises(TypeError):
QnAMaker(None)
def test_set_default_options_with_no_options_arg(self):
qna_without_options = QnAMaker(self.tests_endpoint)
options = qna_without_options._generate_answer_helper.options
default_threshold = 0.3
default_top = 1
default_strict_filters = []
self.assertEqual(default_threshold, options.score_threshold)
self.assertEqual(default_top, options.top)
self.assertEqual(default_strict_filters, options.strict_filters)
def test_options_passed_to_ctor(self):
options = QnAMakerOptions(
score_threshold=0.8,
timeout=9000,
top=5,
strict_filters=[Metadata("movie", "disney")],
)
qna_with_options = QnAMaker(self.tests_endpoint, options)
actual_options = qna_with_options._generate_answer_helper.options
expected_threshold = 0.8
expected_timeout = 9000
expected_top = 5
expected_strict_filters = [Metadata("movie", "disney")]
self.assertEqual(expected_threshold, actual_options.score_threshold)
self.assertEqual(expected_timeout, actual_options.timeout)
self.assertEqual(expected_top, actual_options.top)
self.assertEqual(expected_strict_filters[0].name,
actual_options.strict_filters[0].name)
self.assertEqual(expected_strict_filters[0].value,
actual_options.strict_filters[0].value)
async def test_returns_answer(self):
# Arrange
question: str = "how do I clean the stove?"
response_path: str = "ReturnsAnswer.json"
# Act
result = await QnaApplicationTest._get_service_result(
question, response_path)
first_answer = result[0]
# Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
self.assertEqual(
"BaseCamp: You can use a damp rag to clean around the Power Pack",
first_answer.answer,
)
async def test_returns_answer_using_options(self):
# Arrange
question: str = "up"
response_path: str = "AnswerWithOptions.json"
options = QnAMakerOptions(score_threshold=0.8,
top=5,
strict_filters=[Metadata("movie", "disney")])
# Act
result = await QnaApplicationTest._get_service_result(question,
response_path,
options=options)
first_answer = result[0]
has_at_least_1_ans = True
first_metadata = first_answer.metadata[0]
# Assert
self.assertIsNotNone(result)
self.assertEqual(has_at_least_1_ans, len(result) >= 1)
self.assertTrue(first_answer.answer[0])
self.assertEqual("is a movie", first_answer.answer)
self.assertTrue(first_answer.score >= options.score_threshold)
self.assertEqual("movie", first_metadata.name)
self.assertEqual("disney", first_metadata.value)
async def test_trace_test(self):
activity = Activity(
type=ActivityTypes.message,
text="how do I clean the stove?",
conversation=ConversationAccount(),
recipient=ChannelAccount(),
from_property=ChannelAccount(),
)
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
context = TestContext(activity)
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
result = await qna.get_answers(context)
qna_trace_activities = list(
filter(
lambda act: act.type == "trace" and act.name == "QnAMaker",
context.sent,
))
trace_activity = qna_trace_activities[0]
self.assertEqual("trace", trace_activity.type)
self.assertEqual("QnAMaker", trace_activity.name)
self.assertEqual("QnAMaker Trace", trace_activity.label)
self.assertEqual("https://www.qnamaker.ai/schemas/trace",
trace_activity.value_type)
self.assertEqual(True, hasattr(trace_activity, "value"))
self.assertEqual(True, hasattr(trace_activity.value, "message"))
self.assertEqual(True,
hasattr(trace_activity.value, "query_results"))
self.assertEqual(True,
hasattr(trace_activity.value, "score_threshold"))
self.assertEqual(True, hasattr(trace_activity.value, "top"))
self.assertEqual(True,
hasattr(trace_activity.value, "strict_filters"))
self.assertEqual(self._knowledge_base_id,
trace_activity.value.knowledge_base_id)
return result
async def test_returns_answer_with_timeout(self):
question: str = "how do I clean the stove?"
options = QnAMakerOptions(timeout=999999)
qna = QnAMaker(QnaApplicationTest.tests_endpoint, options)
context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
result = await qna.get_answers(context, options)
self.assertIsNotNone(result)
self.assertEqual(options.timeout,
qna._generate_answer_helper.options.timeout)
async def test_telemetry_returns_answer(self):
# Arrange
question: str = "how do I clean the stove?"
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = True
context = QnaApplicationTest._get_context(question, TestAdapter())
qna = QnAMaker(
QnaApplicationTest.tests_endpoint,
telemetry_client=telemetry_client,
log_personal_information=log_personal_information,
)
# Act
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context)
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args["properties"]
telemetry_metrics = telemetry_args["measurements"]
number_of_args = len(telemetry_args)
first_answer = telemetry_args["properties"][
QnATelemetryConstants.answer_property]
expected_answer = (
"BaseCamp: You can use a damp rag to clean around the Power Pack"
)
# Assert - Check Telemetry logged.
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, number_of_args)
self.assertEqual("QnaMessage", telemetry_args["name"])
self.assertTrue("answer" in telemetry_properties)
self.assertTrue("knowledgeBaseId" in telemetry_properties)
self.assertTrue("matchedQuestion" in telemetry_properties)
self.assertTrue("question" in telemetry_properties)
self.assertTrue("questionId" in telemetry_properties)
self.assertTrue("articleFound" in telemetry_properties)
self.assertEqual(expected_answer, first_answer)
self.assertTrue("score" in telemetry_metrics)
self.assertEqual(1, telemetry_metrics["score"])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer)
self.assertEqual("Editorial", results[0].source)
async def test_telemetry_returns_answer_when_no_answer_found_in_kb(self):
# Arrange
question: str = "gibberish question"
response_json = QnaApplicationTest._get_json_for_file(
"NoAnswerFoundInKb.json")
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
qna = QnAMaker(
QnaApplicationTest.tests_endpoint,
telemetry_client=telemetry_client,
log_personal_information=True,
)
context = QnaApplicationTest._get_context(question, TestAdapter())
# Act
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context)
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args["properties"]
number_of_args = len(telemetry_args)
first_answer = telemetry_args["properties"][
QnATelemetryConstants.answer_property]
expected_answer = "No Qna Answer matched"
expected_matched_question = "No Qna Question matched"
# Assert - Check Telemetry logged.
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, number_of_args)
self.assertEqual("QnaMessage", telemetry_args["name"])
self.assertTrue("answer" in telemetry_properties)
self.assertTrue("knowledgeBaseId" in telemetry_properties)
self.assertTrue("matchedQuestion" in telemetry_properties)
self.assertEqual(
expected_matched_question,
telemetry_properties[
QnATelemetryConstants.matched_question_property],
)
self.assertTrue("question" in telemetry_properties)
self.assertTrue("questionId" in telemetry_properties)
self.assertTrue("articleFound" in telemetry_properties)
self.assertEqual(expected_answer, first_answer)
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(0, len(results))
async def test_telemetry_pii(self):
# Arrange
question: str = "how do I clean the stove?"
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
context = QnaApplicationTest._get_context(question, TestAdapter())
qna = QnAMaker(
QnaApplicationTest.tests_endpoint,
telemetry_client=telemetry_client,
log_personal_information=log_personal_information,
)
# Act
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context)
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args["properties"]
telemetry_metrics = telemetry_args["measurements"]
number_of_args = len(telemetry_args)
first_answer = telemetry_args["properties"][
QnATelemetryConstants.answer_property]
expected_answer = (
"BaseCamp: You can use a damp rag to clean around the Power Pack"
)
# Assert - Validate PII properties not logged.
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, number_of_args)
self.assertEqual("QnaMessage", telemetry_args["name"])
self.assertTrue("answer" in telemetry_properties)
self.assertTrue("knowledgeBaseId" in telemetry_properties)
self.assertTrue("matchedQuestion" in telemetry_properties)
self.assertTrue("question" not in telemetry_properties)
self.assertTrue("questionId" in telemetry_properties)
self.assertTrue("articleFound" in telemetry_properties)
self.assertEqual(expected_answer, first_answer)
self.assertTrue("score" in telemetry_metrics)
self.assertEqual(1, telemetry_metrics["score"])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer)
self.assertEqual("Editorial", results[0].source)
async def test_telemetry_override(self):
# Arrange
question: str = "how do I clean the stove?"
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
context = QnaApplicationTest._get_context(question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act - Override the QnAMaker object to log custom stuff and honor params passed in.
telemetry_properties: Dict[str, str] = {"id": "MyId"}
qna = QnaApplicationTest.OverrideTelemetry(
QnaApplicationTest.tests_endpoint,
options,
None,
telemetry_client,
log_personal_information,
)
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context, options,
telemetry_properties)
telemetry_args = telemetry_client.track_event.call_args_list
first_call_args = telemetry_args[0][0]
first_call_properties = first_call_args[1]
second_call_args = telemetry_args[1][0]
second_call_properties = second_call_args[1]
expected_answer = (
"BaseCamp: You can use a damp rag to clean around the Power Pack"
)
# Assert
self.assertEqual(2, telemetry_client.track_event.call_count)
self.assertEqual(2, len(first_call_args))
self.assertEqual("QnaMessage", first_call_args[0])
self.assertEqual(2, len(first_call_properties))
self.assertTrue("my_important_property" in first_call_properties)
self.assertEqual("my_important_value",
first_call_properties["my_important_property"])
self.assertTrue("id" in first_call_properties)
self.assertEqual("MyId", first_call_properties["id"])
self.assertEqual("my_second_event", second_call_args[0])
self.assertTrue("my_important_property2" in second_call_properties)
self.assertEqual("my_important_value2",
second_call_properties["my_important_property2"])
# Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer)
self.assertEqual("Editorial", results[0].source)
async def test_telemetry_additional_props_metrics(self):
# Arrange
question: str = "how do I clean the stove?"
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
context = QnaApplicationTest._get_context(question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
qna = QnAMaker(
QnaApplicationTest.tests_endpoint,
options,
None,
telemetry_client,
log_personal_information,
)
telemetry_properties: Dict[str, str] = {
"my_important_property": "my_important_value"
}
telemetry_metrics: Dict[str, float] = {
"my_important_metric": 3.14159
}
results = await qna.get_answers(context, None,
telemetry_properties,
telemetry_metrics)
# Assert - Added properties were added.
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args["properties"]
expected_answer = (
"BaseCamp: You can use a damp rag to clean around the Power Pack"
)
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, len(telemetry_args))
self.assertEqual("QnaMessage", telemetry_args["name"])
self.assertTrue("knowledgeBaseId" in telemetry_properties)
self.assertTrue("question" not in telemetry_properties)
self.assertTrue("matchedQuestion" in telemetry_properties)
self.assertTrue("questionId" in telemetry_properties)
self.assertTrue("answer" in telemetry_properties)
self.assertTrue(expected_answer, telemetry_properties["answer"])
self.assertTrue("my_important_property" in telemetry_properties)
self.assertEqual("my_important_value",
telemetry_properties["my_important_property"])
tracked_metrics = telemetry_args["measurements"]
self.assertEqual(2, len(tracked_metrics))
self.assertTrue("score" in tracked_metrics)
self.assertTrue("my_important_metric" in tracked_metrics)
self.assertEqual(3.14159, tracked_metrics["my_important_metric"])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer)
self.assertEqual("Editorial", results[0].source)
async def test_telemetry_additional_props_override(self):
question: str = "how do I clean the stove?"
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
context = QnaApplicationTest._get_context(question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act - Pass in properties during QnA invocation that override default properties
# NOTE: We are invoking this with PII turned OFF, and passing a PII property (originalQuestion).
qna = QnAMaker(
QnaApplicationTest.tests_endpoint,
options,
None,
telemetry_client,
log_personal_information,
)
telemetry_properties = {
"knowledge_base_id": "my_important_value",
"original_question": "my_important_value2",
}
telemetry_metrics = {"score": 3.14159}
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context, None,
telemetry_properties,
telemetry_metrics)
# Assert - Added properties were added.
tracked_args = telemetry_client.track_event.call_args_list[0][1]
tracked_properties = tracked_args["properties"]
expected_answer = (
"BaseCamp: You can use a damp rag to clean around the Power Pack"
)
tracked_metrics = tracked_args["measurements"]
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, len(tracked_args))
self.assertEqual("QnaMessage", tracked_args["name"])
self.assertTrue("knowledge_base_id" in tracked_properties)
self.assertEqual("my_important_value",
tracked_properties["knowledge_base_id"])
self.assertTrue("original_question" in tracked_properties)
self.assertTrue("matchedQuestion" in tracked_properties)
self.assertEqual("my_important_value2",
tracked_properties["original_question"])
self.assertTrue("question" not in tracked_properties)
self.assertTrue("questionId" in tracked_properties)
self.assertTrue("answer" in tracked_properties)
self.assertEqual(expected_answer, tracked_properties["answer"])
self.assertTrue("my_important_property" not in tracked_properties)
self.assertEqual(1, len(tracked_metrics))
self.assertTrue("score" in tracked_metrics)
self.assertEqual(3.14159, tracked_metrics["score"])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer)
self.assertEqual("Editorial", results[0].source)
async def test_telemetry_fill_props_override(self):
# Arrange
question: str = "how do I clean the stove?"
response_json = QnaApplicationTest._get_json_for_file(
"ReturnsAnswer.json")
context: TurnContext = QnaApplicationTest._get_context(
question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act - Pass in properties during QnA invocation that override default properties
# In addition Override with derivation. This presents an interesting question of order of setting
# properties.
# If I want to override "originalQuestion" property:
# - Set in "Stock" schema
# - Set in derived QnAMaker class
# - Set in GetAnswersAsync
# Logically, the GetAnswersAync should win. But ultimately OnQnaResultsAsync decides since it is the last
# code to touch the properties before logging (since it actually logs the event).
qna = QnaApplicationTest.OverrideFillTelemetry(
QnaApplicationTest.tests_endpoint,
options,
None,
telemetry_client,
log_personal_information,
)
telemetry_properties: Dict[str, str] = {
"knowledgeBaseId": "my_important_value",
"matchedQuestion": "my_important_value2",
}
telemetry_metrics: Dict[str, float] = {"score": 3.14159}
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context, None,
telemetry_properties,
telemetry_metrics)
# Assert - Added properties were added.
first_call_args = telemetry_client.track_event.call_args_list[0][0]
first_properties = first_call_args[1]
expected_answer = (
"BaseCamp: You can use a damp rag to clean around the Power Pack"
)
first_metrics = first_call_args[2]
self.assertEqual(2, telemetry_client.track_event.call_count)
self.assertEqual(3, len(first_call_args))
self.assertEqual("QnaMessage", first_call_args[0])
self.assertEqual(6, len(first_properties))
self.assertTrue("knowledgeBaseId" in first_properties)
self.assertEqual("my_important_value",
first_properties["knowledgeBaseId"])
self.assertTrue("matchedQuestion" in first_properties)
self.assertEqual("my_important_value2",
first_properties["matchedQuestion"])
self.assertTrue("questionId" in first_properties)
self.assertTrue("answer" in first_properties)
self.assertEqual(expected_answer, first_properties["answer"])
self.assertTrue("articleFound" in first_properties)
self.assertTrue("my_important_property" in first_properties)
self.assertEqual("my_important_value",
first_properties["my_important_property"])
self.assertEqual(1, len(first_metrics))
self.assertTrue("score" in first_metrics)
self.assertEqual(3.14159, first_metrics["score"])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer)
self.assertEqual("Editorial", results[0].source)
async def test_call_train(self):
feedback_records = []
feedback1 = FeedbackRecord(qna_id=1,
user_id="test",
user_question="How are you?")
feedback2 = FeedbackRecord(qna_id=2,
user_id="test",
user_question="What up??")
feedback_records.extend([feedback1, feedback2])
with patch.object(QnAMaker, "call_train",
return_value=None) as mocked_call_train:
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
qna.call_train(feedback_records)
mocked_call_train.assert_called_once_with(feedback_records)
async def test_should_filter_low_score_variation(self):
options = QnAMakerOptions(top=5)
qna = QnAMaker(QnaApplicationTest.tests_endpoint, options)
question: str = "Q11"
context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
"TopNAnswer.json")
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(context)
self.assertEqual(4, len(results),
"Should have received 4 answers.")
filtered_results = qna.get_low_score_variation(results)
self.assertEqual(
3,
len(filtered_results),
"Should have 3 filtered answers after low score variation.",
)
async def test_should_answer_with_prompts(self):
options = QnAMakerOptions(top=2)
qna = QnAMaker(QnaApplicationTest.tests_endpoint, options)
question: str = "how do I clean the stove?"
turn_context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
"AnswerWithPrompts.json")
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(turn_context, options)
self.assertEqual(1, len(results),
"Should have received 1 answers.")
self.assertEqual(1, len(results[0].context.prompts),
"Should have received 1 prompt.")
async def test_should_answer_with_high_score_provided_context(self):
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
question: str = "where can I buy?"
context = QnARequestContext(
previous_qna_id=5, prvious_user_query="how do I clean the stove?")
options = QnAMakerOptions(top=2, qna_id=55, context=context)
turn_context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
"AnswerWithHighScoreProvidedContext.json")
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(turn_context, options)
self.assertEqual(1, len(results),
"Should have received 1 answers.")
self.assertEqual(1, results[0].score, "Score should be high.")
async def test_should_answer_with_high_score_provided_qna_id(self):
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
question: str = "where can I buy?"
options = QnAMakerOptions(top=2, qna_id=55)
turn_context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
"AnswerWithHighScoreProvidedContext.json")
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(turn_context, options)
self.assertEqual(1, len(results),
"Should have received 1 answers.")
self.assertEqual(1, results[0].score, "Score should be high.")
async def test_should_answer_with_low_score_without_provided_context(self):
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
question: str = "where can I buy?"
options = QnAMakerOptions(top=2, context=None)
turn_context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
"AnswerWithLowScoreProvidedWithoutContext.json")
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
results = await qna.get_answers(turn_context, options)
self.assertEqual(2, len(results),
"Should have received more than one answers.")
self.assertEqual(True, results[0].score < 1,
"Score should be low.")
@classmethod
async def _get_service_result(
cls,
utterance: str,
response_file: str,
bot_adapter: BotAdapter = TestAdapter(),
options: QnAMakerOptions = None,
) -> [dict]:
response_json = QnaApplicationTest._get_json_for_file(response_file)
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
context = QnaApplicationTest._get_context(utterance, bot_adapter)
with patch(
"aiohttp.ClientSession.post",
return_value=aiounittest.futurized(response_json),
):
result = await qna.get_answers(context, options)
return result
@classmethod
def _get_json_for_file(cls, response_file: str) -> object:
curr_dir = path.dirname(path.abspath(__file__))
response_path = path.join(curr_dir, "test_data", response_file)
with open(response_path, "r", encoding="utf-8-sig") as file:
response_str = file.read()
response_json = json.loads(response_str)
return response_json
@staticmethod
def _get_context(question: str, bot_adapter: BotAdapter) -> TurnContext:
test_adapter = bot_adapter or TestAdapter()
activity = Activity(
type=ActivityTypes.message,
text=question,
conversation=ConversationAccount(),
recipient=ChannelAccount(),
from_property=ChannelAccount(),
)
return TurnContext(test_adapter, activity)
class OverrideTelemetry(QnAMaker):
def __init__( # pylint: disable=useless-super-delegation
self,
endpoint: QnAMakerEndpoint,
options: QnAMakerOptions,
http_client: ClientSession,
telemetry_client: BotTelemetryClient,
log_personal_information: bool,
):
super().__init__(
endpoint,
options,
http_client,
telemetry_client,
log_personal_information,
)
async def on_qna_result( # pylint: disable=unused-argument
self,
query_results: [QueryResult],
turn_context: TurnContext,
telemetry_properties: Dict[str, str] = None,
telemetry_metrics: Dict[str, float] = None,
):
properties = telemetry_properties or {}
# get_answers overrides derived class
properties["my_important_property"] = "my_important_value"
# Log event
self.telemetry_client.track_event(
QnATelemetryConstants.qna_message_event, properties)
# Create 2nd event.
second_event_properties = {
"my_important_property2": "my_important_value2"
}
self.telemetry_client.track_event("my_second_event",
second_event_properties)
class OverrideFillTelemetry(QnAMaker):
def __init__( # pylint: disable=useless-super-delegation
self,
endpoint: QnAMakerEndpoint,
options: QnAMakerOptions,
http_client: ClientSession,
telemetry_client: BotTelemetryClient,
log_personal_information: bool,
):
super().__init__(
endpoint,
options,
http_client,
telemetry_client,
log_personal_information,
)
async def on_qna_result(
self,
query_results: [QueryResult],
turn_context: TurnContext,
telemetry_properties: Dict[str, str] = None,
telemetry_metrics: Dict[str, float] = None,
):
event_data = await self.fill_qna_event(query_results, turn_context,
telemetry_properties,
telemetry_metrics)
# Add my property.
event_data.properties.update(
{"my_important_property": "my_important_value"})
# Log QnaMessage event.
self.telemetry_client.track_event(
QnATelemetryConstants.qna_message_event,
event_data.properties,
event_data.metrics,
)
# Create second event.
second_event_properties: Dict[str, str] = {
"my_important_property2": "my_important_value2"
}
self.telemetry_client.track_event("MySecondEvent",
second_event_properties)
def __init__(self, kb_id: str, endpoint_key: str, host: str):
print(f'kb_id: {kb_id}, endpoint: {endpoint_key}, host: {host}',
file=sys.stderr)
self._qna_endpoint = QnAMakerEndpoint(kb_id, endpoint_key, host)
self._qna_maker = QnAMaker(self._qna_endpoint)
def __init__(self):
qna_enpoint = QnAMakerEndpoint(
"72e97083-2783-4e2c-a3ae-cac7d785a973",
"21a6fbca-8f7f-4ecf-8f47-704ddee75aaa",
"https://yogyabot.azurewebsites.net/qnamaker")
self.qna_maker = QnAMaker(qna_enpoint)
def __init__(self):
qna_endpoint = QnAMakerEndpoint(
"ea9c386a-53e0-4dc4-95d5-ecae7cfa1a42",
"d777e3f4-52d1-4a86-a75d-a97ede3fa509",
"https://testqnamaker001.azurewebsites.net/qnamaker")
self.qna_maker = QnAMaker(qna_endpoint)
def __init__(self):
qna_endpoint = QnAMakerEndpoint(par.kb_id, par.auth_id, par.ep_url)
self.qna_maker = QnAMaker(qna_endpoint)
def test_endpoint_with_empty_kbid(self):
empty_kbid = ""
with self.assertRaises(TypeError):
QnAMakerEndpoint(empty_kbid, self._endpoint_key, self._host)
class QnaApplicationTest(aiounittest.AsyncTestCase):
# Note this is NOT a real LUIS application ID nor a real LUIS subscription-key
# theses are GUIDs edited to look right to the parsing and validation code.
_knowledge_base_id: str = 'f028d9k3-7g9z-11d3-d300-2b8x98227q8w'
_endpoint_key: str = '1k997n7w-207z-36p3-j2u1-09tas20ci6011'
_host: str = 'https://dummyqnahost.azurewebsites.net/qnamaker'
tests_endpoint = QnAMakerEndpoint(_knowledge_base_id, _endpoint_key, _host)
def test_qnamaker_construction(self):
# Arrange
endpoint = self.tests_endpoint
# Act
qna = QnAMaker(endpoint)
endpoint = qna._endpoint
# Assert
self.assertEqual('f028d9k3-7g9z-11d3-d300-2b8x98227q8w',
endpoint.knowledge_base_id)
self.assertEqual('1k997n7w-207z-36p3-j2u1-09tas20ci6011',
endpoint.endpoint_key)
self.assertEqual('https://dummyqnahost.azurewebsites.net/qnamaker',
endpoint.host)
def test_endpoint_with_empty_kbid(self):
empty_kbid = ''
with self.assertRaises(TypeError):
QnAMakerEndpoint(empty_kbid, self._endpoint_key, self._host)
def test_endpoint_with_empty_endpoint_key(self):
empty_endpoint_key = ''
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, empty_endpoint_key,
self._host)
def test_endpoint_with_emptyhost(self):
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, self._endpoint_key, '')
def test_qnamaker_with_none_endpoint(self):
with self.assertRaises(TypeError):
QnAMaker(None)
def test_v2_legacy_endpoint(self):
v2_hostname = 'https://westus.api.cognitive.microsoft.com/qnamaker/v2.0'
v2_legacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v2_hostname)
with self.assertRaises(ValueError):
QnAMaker(v2_legacy_endpoint)
def test_legacy_protocol(self):
v3_hostname = 'https://westus.api.cognitive.microsoft.com/qnamaker/v3.0'
v3_legacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v3_hostname)
legacy_qna = QnAMaker(v3_legacy_endpoint)
is_legacy = True
v4_hostname = 'https://UpdatedNonLegacyQnaHostName.azurewebsites.net/qnamaker'
nonlegacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v4_hostname)
v4_qna = QnAMaker(nonlegacy_endpoint)
self.assertEqual(is_legacy, legacy_qna._is_legacy_protocol)
self.assertNotEqual(is_legacy, v4_qna._is_legacy_protocol)
def test_set_default_options_with_no_options_arg(self):
qna_without_options = QnAMaker(self.tests_endpoint)
options = qna_without_options._options
default_threshold = 0.3
default_top = 1
default_strict_filters = []
self.assertEqual(default_threshold, options.score_threshold)
self.assertEqual(default_top, options.top)
self.assertEqual(default_strict_filters, options.strict_filters)
def test_options_passed_to_ctor(self):
options = QnAMakerOptions(score_threshold=0.8,
timeout=9000,
top=5,
strict_filters=[Metadata('movie', 'disney')])
qna_with_options = QnAMaker(self.tests_endpoint, options)
actual_options = qna_with_options._options
expected_threshold = 0.8
expected_timeout = 9000
expected_top = 5
expected_strict_filters = [Metadata('movie', 'disney')]
self.assertEqual(expected_threshold, actual_options.score_threshold)
self.assertEqual(expected_timeout, actual_options.timeout)
self.assertEqual(expected_top, actual_options.top)
self.assertEqual(expected_strict_filters[0].name,
actual_options.strict_filters[0].name)
self.assertEqual(expected_strict_filters[0].value,
actual_options.strict_filters[0].value)
async def test_returns_answer(self):
# Arrange
question: str = 'how do I clean the stove?'
response_path: str = 'ReturnsAnswer.json'
# Act
result = await QnaApplicationTest._get_service_result(
question, response_path)
first_answer = result[0]
#Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result))
self.assertEqual(
'BaseCamp: You can use a damp rag to clean around the Power Pack',
first_answer.answer[0])
async def test_returns_answer_using_options(self):
# Arrange
question: str = 'up'
response_path: str = 'AnswerWithOptions.json'
options = QnAMakerOptions(score_threshold=0.8,
top=5,
strict_filters=[{
'name': 'movie',
'value': 'disney'
}])
# Act
result = await QnaApplicationTest._get_service_result(question,
response_path,
options=options)
first_answer = result[0]
has_at_least_1_ans = True
first_metadata = first_answer.metadata[0][0]
# Assert
self.assertIsNotNone(result)
self.assertEqual(has_at_least_1_ans, len(result) >= 1)
self.assertTrue(first_answer.answer[0])
self.assertEqual('is a movie', first_answer.answer[0])
self.assertTrue(first_answer.score[0] >= options.score_threshold)
self.assertEqual('movie', first_metadata.name)
self.assertEqual('disney', first_metadata.value)
async def test_trace_test(self):
activity = Activity(
type=ActivityTypes.message,
text='how do I clean the stove?',
conversation=ConversationAccount(),
recipient=ChannelAccount(),
from_property=ChannelAccount(),
)
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
context = TestContext(activity)
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
result = await qna.get_answers(context)
qna_trace_activities = list(
filter(
lambda act: act.type == 'trace' and act.name == 'QnAMaker',
context.sent))
trace_activity = qna_trace_activities[0]
self.assertEqual('trace', trace_activity.type)
self.assertEqual('QnAMaker', trace_activity.name)
self.assertEqual('QnAMaker Trace', trace_activity.label)
self.assertEqual('https://www.qnamaker.ai/schemas/trace',
trace_activity.value_type)
self.assertEqual(True, hasattr(trace_activity, 'value'))
self.assertEqual(True, hasattr(trace_activity.value, 'message'))
self.assertEqual(True,
hasattr(trace_activity.value, 'query_results'))
self.assertEqual(True,
hasattr(trace_activity.value, 'score_threshold'))
self.assertEqual(True, hasattr(trace_activity.value, 'top'))
self.assertEqual(True,
hasattr(trace_activity.value, 'strict_filters'))
self.assertEqual(self._knowledge_base_id,
trace_activity.value.knowledge_base_id[0])
return result
async def test_returns_answer_with_timeout(self):
question: str = 'how do I clean the stove?'
options = QnAMakerOptions(timeout=999999)
qna = QnAMaker(QnaApplicationTest.tests_endpoint, options)
context = QnaApplicationTest._get_context(question, TestAdapter())
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
result = await qna.get_answers(context, options)
self.assertIsNotNone(result)
self.assertEqual(options.timeout, qna._options.timeout)
async def test_telemetry_returns_answer(self):
# Arrange
question: str = 'how do I clean the stove?'
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = True
context = QnaApplicationTest._get_context(question, TestAdapter())
qna = QnAMaker(QnaApplicationTest.tests_endpoint,
telemetry_client=telemetry_client,
log_personal_information=log_personal_information)
# Act
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
results = await qna.get_answers(context)
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args['properties']
telemetry_metrics = telemetry_args['measurements']
number_of_args = len(telemetry_args)
first_answer = telemetry_args['properties'][
QnATelemetryConstants.answer_property][0]
expected_answer = 'BaseCamp: You can use a damp rag to clean around the Power Pack'
# Assert - Check Telemetry logged.
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, number_of_args)
self.assertEqual('QnaMessage', telemetry_args['name'])
self.assertTrue('answer' in telemetry_properties)
self.assertTrue('knowledgeBaseId' in telemetry_properties)
self.assertTrue('matchedQuestion' in telemetry_properties)
self.assertTrue('question' in telemetry_properties)
self.assertTrue('questionId' in telemetry_properties)
self.assertTrue('articleFound' in telemetry_properties)
self.assertEqual(expected_answer, first_answer)
self.assertTrue('score' in telemetry_metrics)
self.assertEqual(1, telemetry_metrics['score'][0])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer[0])
self.assertEqual('Editorial', results[0].source)
async def test_telemetry_returns_answer_when_no_answer_found_in_kb(self):
# Arrange
question: str = 'gibberish question'
response_json = QnaApplicationTest._get_json_for_file(
'NoAnswerFoundInKb.json')
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
qna = QnAMaker(QnaApplicationTest.tests_endpoint,
telemetry_client=telemetry_client,
log_personal_information=True)
context = QnaApplicationTest._get_context(question, TestAdapter())
# Act
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
results = await qna.get_answers(context)
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args['properties']
number_of_args = len(telemetry_args)
first_answer = telemetry_args['properties'][
QnATelemetryConstants.answer_property]
expected_answer = 'No Qna Answer matched'
expected_matched_question = 'No Qna Question matched'
# Assert - Check Telemetry logged.
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, number_of_args)
self.assertEqual('QnaMessage', telemetry_args['name'])
self.assertTrue('answer' in telemetry_properties)
self.assertTrue('knowledgeBaseId' in telemetry_properties)
self.assertTrue('matchedQuestion' in telemetry_properties)
self.assertEqual(
expected_matched_question, telemetry_properties[
QnATelemetryConstants.matched_question_property])
self.assertTrue('question' in telemetry_properties)
self.assertTrue('questionId' in telemetry_properties)
self.assertTrue('articleFound' in telemetry_properties)
self.assertEqual(expected_answer, first_answer)
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(0, len(results))
async def test_telemetry_pii(self):
# Arrange
question: str = 'how do I clean the stove?'
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
context = QnaApplicationTest._get_context(question, TestAdapter())
qna = QnAMaker(QnaApplicationTest.tests_endpoint,
telemetry_client=telemetry_client,
log_personal_information=log_personal_information)
# Act
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
results = await qna.get_answers(context)
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args['properties']
telemetry_metrics = telemetry_args['measurements']
number_of_args = len(telemetry_args)
first_answer = telemetry_args['properties'][
QnATelemetryConstants.answer_property][0]
expected_answer = 'BaseCamp: You can use a damp rag to clean around the Power Pack'
# Assert - Validate PII properties not logged.
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, number_of_args)
self.assertEqual('QnaMessage', telemetry_args['name'])
self.assertTrue('answer' in telemetry_properties)
self.assertTrue('knowledgeBaseId' in telemetry_properties)
self.assertTrue('matchedQuestion' in telemetry_properties)
self.assertTrue('question' not in telemetry_properties)
self.assertTrue('questionId' in telemetry_properties)
self.assertTrue('articleFound' in telemetry_properties)
self.assertEqual(expected_answer, first_answer)
self.assertTrue('score' in telemetry_metrics)
self.assertEqual(1, telemetry_metrics['score'][0])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer[0])
self.assertEqual('Editorial', results[0].source)
async def test_telemetry_override(self):
# Arrange
question: str = 'how do I clean the stove?'
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
context = QnaApplicationTest._get_context(question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act - Override the QnAMaker object to log custom stuff and honor params passed in.
telemetry_properties: Dict[str, str] = {'id': 'MyId'}
qna = QnaApplicationTest.OverrideTelemetry(
QnaApplicationTest.tests_endpoint, options, None, telemetry_client,
log_personal_information)
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
results = await qna.get_answers(context, options,
telemetry_properties)
telemetry_args = telemetry_client.track_event.call_args_list
first_call_args = telemetry_args[0][0]
first_call_properties = first_call_args[1]
second_call_args = telemetry_args[1][0]
second_call_properties = second_call_args[1]
expected_answer = 'BaseCamp: You can use a damp rag to clean around the Power Pack'
# Assert
self.assertEqual(2, telemetry_client.track_event.call_count)
self.assertEqual(2, len(first_call_args))
self.assertEqual('QnaMessage', first_call_args[0])
self.assertEqual(2, len(first_call_properties))
self.assertTrue('my_important_property' in first_call_properties)
self.assertEqual('my_important_value',
first_call_properties['my_important_property'])
self.assertTrue('id' in first_call_properties)
self.assertEqual('MyId', first_call_properties['id'])
self.assertEqual('my_second_event', second_call_args[0])
self.assertTrue('my_important_property2' in second_call_properties)
self.assertEqual('my_important_value2',
second_call_properties['my_important_property2'])
# Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer[0])
self.assertEqual('Editorial', results[0].source)
async def test_telemetry_additional_props_metrics(self):
# Arrange
question: str = 'how do I clean the stove?'
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
context = QnaApplicationTest._get_context(question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
qna = QnAMaker(QnaApplicationTest.tests_endpoint, options, None,
telemetry_client, log_personal_information)
telemetry_properties: Dict[str, str] = {
'my_important_property': 'my_important_value'
}
telemetry_metrics: Dict[str, float] = {
'my_important_metric': 3.14159
}
results = await qna.get_answers(context, None,
telemetry_properties,
telemetry_metrics)
# Assert - Added properties were added.
telemetry_args = telemetry_client.track_event.call_args_list[0][1]
telemetry_properties = telemetry_args['properties']
expected_answer = 'BaseCamp: You can use a damp rag to clean around the Power Pack'
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, len(telemetry_args))
self.assertEqual('QnaMessage', telemetry_args['name'])
self.assertTrue('knowledgeBaseId' in telemetry_properties)
self.assertTrue('question' not in telemetry_properties)
self.assertTrue('matchedQuestion' in telemetry_properties)
self.assertTrue('questionId' in telemetry_properties)
self.assertTrue('answer' in telemetry_properties)
self.assertTrue(expected_answer, telemetry_properties['answer'][0])
self.assertTrue('my_important_property' in telemetry_properties)
self.assertEqual('my_important_value',
telemetry_properties['my_important_property'])
tracked_metrics = telemetry_args['measurements']
self.assertEqual(2, len(tracked_metrics))
self.assertTrue('score' in tracked_metrics)
self.assertTrue('my_important_metric' in tracked_metrics)
self.assertEqual(3.14159, tracked_metrics['my_important_metric'])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer[0])
self.assertEqual('Editorial', results[0].source)
async def test_telemetry_additional_props_override(self):
question: str = 'how do I clean the stove?'
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
context = QnaApplicationTest._get_context(question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act - Pass in properties during QnA invocation that override default properties
# NOTE: We are invoking this with PII turned OFF, and passing a PII property (originalQuestion).
qna = QnAMaker(QnaApplicationTest.tests_endpoint, options, None,
telemetry_client, log_personal_information)
telemetry_properties = {
'knowledge_base_id': 'my_important_value',
'original_question': 'my_important_value2'
}
telemetry_metrics = {'score': 3.14159}
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
results = await qna.get_answers(context, None,
telemetry_properties,
telemetry_metrics)
# Assert - Added properties were added.
tracked_args = telemetry_client.track_event.call_args_list[0][1]
tracked_properties = tracked_args['properties']
expected_answer = 'BaseCamp: You can use a damp rag to clean around the Power Pack'
tracked_metrics = tracked_args['measurements']
self.assertEqual(1, telemetry_client.track_event.call_count)
self.assertEqual(3, len(tracked_args))
self.assertEqual('QnaMessage', tracked_args['name'])
self.assertTrue('knowledge_base_id' in tracked_properties)
self.assertEqual('my_important_value',
tracked_properties['knowledge_base_id'])
self.assertTrue('original_question' in tracked_properties)
self.assertTrue('matchedQuestion' in tracked_properties)
self.assertEqual('my_important_value2',
tracked_properties['original_question'])
self.assertTrue('question' not in tracked_properties)
self.assertTrue('questionId' in tracked_properties)
self.assertTrue('answer' in tracked_properties)
self.assertEqual(expected_answer, tracked_properties['answer'][0])
self.assertTrue('my_important_property' not in tracked_properties)
self.assertEqual(1, len(tracked_metrics))
self.assertTrue('score' in tracked_metrics)
self.assertEqual(3.14159, tracked_metrics['score'])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer[0])
self.assertEqual('Editorial', results[0].source)
async def test_telemetry_fill_props_override(self):
# Arrange
question: str = 'how do I clean the stove?'
response_json = QnaApplicationTest._get_json_for_file(
'ReturnsAnswer.json')
context: TurnContext = QnaApplicationTest._get_context(
question, TestAdapter())
options = QnAMakerOptions(top=1)
telemetry_client = unittest.mock.create_autospec(BotTelemetryClient)
log_personal_information = False
# Act - Pass in properties during QnA invocation that override default properties
# In addition Override with derivation. This presents an interesting question of order of setting properties.
# If I want to override "originalQuestion" property:
# - Set in "Stock" schema
# - Set in derived QnAMaker class
# - Set in GetAnswersAsync
# Logically, the GetAnswersAync should win. But ultimately OnQnaResultsAsync decides since it is the last
# code to touch the properties before logging (since it actually logs the event).
qna = QnaApplicationTest.OverrideFillTelemetry(
QnaApplicationTest.tests_endpoint, options, None, telemetry_client,
log_personal_information)
telemetry_properties: Dict[str, str] = {
'knowledgeBaseId': 'my_important_value',
'matchedQuestion': 'my_important_value2'
}
telemetry_metrics: Dict[str, float] = {'score': 3.14159}
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
results = await qna.get_answers(context, None,
telemetry_properties,
telemetry_metrics)
# Assert - Added properties were added.
first_call_args = telemetry_client.track_event.call_args_list[0][0]
first_properties = first_call_args[1]
expected_answer = 'BaseCamp: You can use a damp rag to clean around the Power Pack'
first_metrics = first_call_args[2]
self.assertEqual(2, telemetry_client.track_event.call_count)
self.assertEqual(3, len(first_call_args))
self.assertEqual('QnaMessage', first_call_args[0])
self.assertEqual(6, len(first_properties))
self.assertTrue('knowledgeBaseId' in first_properties)
self.assertEqual('my_important_value',
first_properties['knowledgeBaseId'])
self.assertTrue('matchedQuestion' in first_properties)
self.assertEqual('my_important_value2',
first_properties['matchedQuestion'])
self.assertTrue('questionId' in first_properties)
self.assertTrue('answer' in first_properties)
self.assertEqual(expected_answer, first_properties['answer'][0])
self.assertTrue('articleFound' in first_properties)
self.assertTrue('my_important_property' in first_properties)
self.assertEqual('my_important_value',
first_properties['my_important_property'])
self.assertEqual(1, len(first_metrics))
self.assertTrue('score' in first_metrics)
self.assertEqual(3.14159, first_metrics['score'])
# Assert - Validate we didn't break QnA functionality.
self.assertIsNotNone(results)
self.assertEqual(1, len(results))
self.assertEqual(expected_answer, results[0].answer[0])
self.assertEqual('Editorial', results[0].source)
@classmethod
async def _get_service_result(cls,
utterance: str,
response_file: str,
bot_adapter: BotAdapter = TestAdapter(),
options: QnAMakerOptions = None) -> [dict]:
response_json = QnaApplicationTest._get_json_for_file(response_file)
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
context = QnaApplicationTest._get_context(utterance, bot_adapter)
with patch('aiohttp.ClientSession.post',
return_value=aiounittest.futurized(response_json)):
result = await qna.get_answers(context, options)
return result
@classmethod
def _get_json_for_file(cls, response_file: str) -> object:
curr_dir = path.dirname(path.abspath(__file__))
response_path = path.join(curr_dir, "test_data", response_file)
with open(response_path, "r", encoding="utf-8-sig") as f:
response_str = f.read()
response_json = json.loads(response_str)
return response_json
@staticmethod
def _get_context(question: str, bot_adapter: BotAdapter) -> TurnContext:
test_adapter = bot_adapter or TestAdapter()
activity = Activity(
type=ActivityTypes.message,
text=question,
conversation=ConversationAccount(),
recipient=ChannelAccount(),
from_property=ChannelAccount(),
)
return TurnContext(test_adapter, activity)
class OverrideTelemetry(QnAMaker):
def __init__(self, endpoint: QnAMakerEndpoint,
options: QnAMakerOptions, http_client: ClientSession,
telemetry_client: BotTelemetryClient,
log_personal_information: bool):
super().__init__(endpoint, options, http_client, telemetry_client,
log_personal_information)
async def on_qna_result(self,
query_results: [QueryResult],
context: TurnContext,
telemetry_properties: Dict[str, str] = None,
telemetry_metrics: Dict[str, float] = None):
properties = telemetry_properties or {}
# get_answers overrides derived class
properties['my_important_property'] = 'my_important_value'
# Log event
self.telemetry_client.track_event(
QnATelemetryConstants.qna_message_event, properties)
# Create 2nd event.
second_event_properties = {
'my_important_property2': 'my_important_value2'
}
self.telemetry_client.track_event('my_second_event',
second_event_properties)
class OverrideFillTelemetry(QnAMaker):
def __init__(self, endpoint: QnAMakerEndpoint,
options: QnAMakerOptions, http_client: ClientSession,
telemetry_client: BotTelemetryClient,
log_personal_information: bool):
super().__init__(endpoint, options, http_client, telemetry_client,
log_personal_information)
async def on_qna_result(self,
query_results: [QueryResult],
context: TurnContext,
telemetry_properties: Dict[str, str] = None,
telemetry_metrics: Dict[str, float] = None):
event_data = await self.fill_qna_event(query_results, context,
telemetry_properties,
telemetry_metrics)
# Add my property.
event_data.properties.update(
{'my_important_property': 'my_important_value'})
# Log QnaMessage event.
self.telemetry_client.track_event(
QnATelemetryConstants.qna_message_event, event_data.properties,
event_data.metrics)
# Create second event.
second_event_properties: Dict[str, str] = {
'my_important_property2': 'my_important_value2'
}
self.telemetry_client.track_event('MySecondEvent',
second_event_properties)
def test_endpoint_with_emptyhost(self):
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, self._endpoint_key, "")
class QnaApplicationTest(aiounittest.AsyncTestCase):
_knowledge_base_id: str = 'a090f9f3-2f8e-41d1-a581-4f7a49269a0c'
_endpoint_key: str = '4a439d5b-163b-47c3-b1d1-168cc0db5608'
_host: str = 'https://ashleyNlpBot1-qnahost.azurewebsites.net/qnamaker'
tests_endpoint = QnAMakerEndpoint(_knowledge_base_id, _endpoint_key, _host)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# self._mocked_results: QueryResult(
# )
def test_qnamaker_construction(self):
# Arrange
endpoint = self.tests_endpoint
# Act
qna = QnAMaker(endpoint)
endpoint = qna._endpoint
# Assert
self.assertEqual('a090f9f3-2f8e-41d1-a581-4f7a49269a0c',
endpoint.knowledge_base_id)
self.assertEqual('4a439d5b-163b-47c3-b1d1-168cc0db5608',
endpoint.endpoint_key)
self.assertEqual(
'https://ashleyNlpBot1-qnahost.azurewebsites.net/qnamaker',
endpoint.host)
def test_endpoint_with_empty_kbid(self):
empty_kbid = ''
with self.assertRaises(TypeError):
QnAMakerEndpoint(empty_kbid, self._endpoint_key, self._host)
def test_endpoint_with_empty_endpoint_key(self):
empty_endpoint_key = ''
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, empty_endpoint_key,
self._host)
def test_endpoint_with_emptyhost(self):
with self.assertRaises(TypeError):
QnAMakerEndpoint(self._knowledge_base_id, self._endpoint_key, '')
def test_qnamaker_with_none_endpoint(self):
with self.assertRaises(TypeError):
QnAMaker(None)
def test_v2_legacy_endpoint(self):
v2_hostname = 'https://westus.api.cognitive.microsoft.com/qnamaker/v2.0'
v2_legacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v2_hostname)
with self.assertRaises(ValueError):
QnAMaker(v2_legacy_endpoint)
def test_legacy_protocol(self):
v3_hostname = 'https://westus.api.cognitive.microsoft.com/qnamaker/v3.0'
v3_legacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v3_hostname)
legacy_qna = QnAMaker(v3_legacy_endpoint)
is_legacy = True
v4_hostname = 'https://UpdatedNonLegacyQnaHostName.azurewebsites.net/qnamaker'
nonlegacy_endpoint = QnAMakerEndpoint(self._knowledge_base_id,
self._endpoint_key, v4_hostname)
v4_qna = QnAMaker(nonlegacy_endpoint)
self.assertEqual(is_legacy, legacy_qna._is_legacy_protocol)
self.assertNotEqual(is_legacy, v4_qna._is_legacy_protocol)
def test_set_default_options_with_no_options_arg(self):
qna_without_options = QnAMaker(self.tests_endpoint)
options = qna_without_options._options
default_threshold = 0.3
default_top = 1
default_strict_filters = []
self.assertEqual(default_threshold, options.score_threshold)
self.assertEqual(default_top, options.top)
self.assertEqual(default_strict_filters, options.strict_filters)
def test_options_passed_to_ctor(self):
options = QnAMakerOptions(score_threshold=0.8,
timeout=9000,
top=5,
strict_filters=[Metadata('movie', 'disney')])
qna_with_options = QnAMaker(self.tests_endpoint, options)
actual_options = qna_with_options._options
expected_threshold = 0.8
expected_timeout = 9000
expected_top = 5
expected_strict_filters = [Metadata('movie', 'disney')]
self.assertEqual(expected_threshold, actual_options.score_threshold)
self.assertEqual(expected_timeout, actual_options.timeout)
self.assertEqual(expected_top, actual_options.top)
self.assertEqual(expected_strict_filters[0].name,
actual_options.strict_filters[0].name)
self.assertEqual(expected_strict_filters[0].value,
actual_options.strict_filters[0].value)
async def test_returns_answer(self):
# Arrange
question: str = 'how do I clean the stove?'
response_path: str = 'ReturnsAnswer.json'
# Act
result = await QnaApplicationTest._get_service_result(
question, response_path)
first_answer = result['answers'][0]
#Assert
self.assertIsNotNone(result)
self.assertEqual(1, len(result['answers']))
self.assertTrue(question in first_answer['questions'])
self.assertEqual(
'BaseCamp: You can use a damp rag to clean around the Power Pack',
first_answer['answer'])
async def test_returns_answer_using_options(self):
# Arrange
question: str = 'up'
response_path: str = 'AnswerWithOptions.json'
options = QnAMakerOptions(score_threshold=0.8,
top=5,
strict_filters=[{
'name': 'movie',
'value': 'disney'
}])
# Act
result = await QnaApplicationTest._get_service_result(question,
response_path,
options=options)
first_answer = result['answers'][0]
has_at_least_1_ans = True
first_metadata = first_answer['metadata'][0]
# Assert
self.assertIsNotNone(result)
self.assertEqual(
has_at_least_1_ans,
len(result['answers']) >= 1
and len(result['answers']) <= options.top)
self.assertTrue(question in first_answer['questions'])
self.assertTrue(first_answer['answer'])
self.assertEqual('is a movie', first_answer['answer'])
self.assertTrue(first_answer['score'] >= options.score_threshold)
self.assertEqual('movie', first_metadata['name'])
self.assertEqual('disney', first_metadata['value'])
@classmethod
async def _get_service_result(cls,
utterance: str,
response_file: str,
bot_adapter: BotAdapter = TestAdapter(),
options: QnAMakerOptions = None) -> [dict]:
response_json = QnaApplicationTest._get_json_for_file(response_file)
qna = QnAMaker(QnaApplicationTest.tests_endpoint)
context = QnaApplicationTest._get_context(utterance, bot_adapter)
response = Mock(spec=Response)
response.status_code = 200
response.headers = {}
response.reason = ''
with patch('requests.post', return_value=response):
with patch('botbuilder.ai.qna.QnAMaker._format_qna_result',
return_value=response_json):
result = await qna.get_answers(context, options)
return result
@classmethod
def _get_json_for_file(cls, response_file: str) -> object:
curr_dir = path.dirname(path.abspath(__file__))
response_path = path.join(curr_dir, "test_data", response_file)
with open(response_path, "r", encoding="utf-8-sig") as f:
response_str = f.read()
response_json = json.loads(response_str)
return response_json
@staticmethod
def _get_context(utterance: str, bot_adapter: BotAdapter) -> TurnContext:
test_adapter = TestAdapter()
activity = Activity(
type=ActivityTypes.message,
text=utterance,
conversation=ConversationAccount(),
recipient=ChannelAccount(),
from_property=ChannelAccount(),
)
return TurnContext(test_adapter, activity)