def setUp(self):
self._scenarios = [
Scenario([
Request('asker', 'hello', 'link', 'https://some_link.com'),
Request('agent', 'world', 'same', ''),
]),
Scenario([
Request('asker', 'question_scenario_2', 'link',
'some_url_scenario_2'),
])
]
self._mock_get_suggestions = Mock()
self._mock_get_suggestions.side_effect = [[
Suggestion('link', 'some_url'),
Suggestion('link', 'another_url')
], [Suggestion('link', 'url_of_a_different_suggestions_response')
], [Suggestion('link', 'some_url_of_suggestion_scenario_2')]]
self._mock_get_time = Mock()
self._mock_get_time.side_effect = [
datetime(2005, 12, 30, 6, 45, 1),
datetime(2005, 12, 30, 6, 45, 2),
datetime(2005, 12, 30, 6, 45, 4),
datetime(2005, 12, 30, 6, 45, 7),
datetime(2005, 12, 30, 6, 45, 11),
datetime(2005, 12, 30, 6, 45, 16)
]
self._scenario_runner = ScenariosRunner(self._mock_get_suggestions,
self._mock_get_time)
def test_fail_when_expected_link_and_no_suggestion_contains_the_link(self):
request = Request(None, None, "link", "expected_link")
suggestions = [
Suggestion("link", "not_the_expected_link"),
Suggestion("question", "question_data")
]
self.assertFalse(
analyse_link_or_question_with_request_data(request, suggestions))
def test_fail_when_expected_question_and_no_suggestion_is_a_question(self):
request = Request(None, None, "question", "")
suggestions = [
Suggestion("link", "someLink"),
Suggestion("link", "someOtherLink")
]
self.assertFalse(
analyse_link_or_question_without_request_data(
request, suggestions))
def test_request_to_data_type_asker(self):
request = Request('asker', 'Hello world', 'To ignore', 'To ignore')
actual_json = get_json_for_whisper_api('11', request, 'myChatKey')
expected_json = {
'chatkey': 'myChatKey',
'query': 'Hello world',
'type': 0
}
self.assertEquals(expected_json, actual_json)
def test_fail_when_expected_notlink_and_suggestion_contains_the_forbidden_link(
self):
request = Request(None, None, "notlink", "forbidden_url")
suggestions = [
Suggestion("link", "urlA"),
Suggestion("link", "forbidden_url"),
Suggestion("link", "urlC")
]
self.assertFalse(
analyse_notlink_with_request_data(request, suggestions))
def test_pass_when_expected_notlink_and_suggestion_does_not_contain_the_forbidden_link(
self):
request = Request(None, None, "notlink", "url_not_in_the_suggestions")
suggestions = [
Suggestion("link", "urlA"),
Suggestion("link", "urlB"),
Suggestion("link", "urlC")
]
self.assertTrue(analyse_notlink_with_request_data(
request, suggestions))
def test_when_not_found_then_get_selected_suggestion_returns_none(self):
suggestions = [
Suggestion("link", "some_dummy_url"),
Suggestion("link", "more_dummy_url"),
Suggestion("link", "another_dummy_url")
]
request = Request(None, None, "link", "some_expected_url")
actual_selected_suggestion = get_selected_suggestion(
suggestions, request)
self.assertIsNone(actual_selected_suggestion)
def test_pass_when_expected_question_and_one_suggestion_is_a_question(
self):
request = Request(None, None, "question", "")
suggestions = [
Suggestion("link", "someLink"),
Suggestion("question", "question_data")
]
self.assertTrue(
analyse_link_or_question_without_request_data(
request, suggestions))
def test_analyze_with_one_successful_suggestions_response(self):
expected_analysis = ['success(1-1)']
request = Request('asker', 'What is Coveo?', 'link',
'https://www.coveo.com/')
suggestion = Suggestion('link', 'https://www.coveo.com/')
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
[Scenario([request])],
[[SuggestionsResponse([suggestion], timedelta(seconds=0))]])
self.assertEquals(expected_analysis,
suggestions_responses_analyzer.analyze_scenarios())
def test_request_to_data_type_agent(self):
request = Request('agent', 'Hello world', 'To ignore', 'To ignore')
actual_json = get_json_for_whisper_api('13', request, 'myChatKey')
expected_json = {
'chatkey': 'myChatKey',
'query': 'Hello world',
'type': 1,
'maxDocuments': 15,
'maxQuestions': 15
}
self.assertEquals(expected_json, actual_json)
def test_pass_when_expected_same_and_suggestions_are_the_same(self):
request = Request(None, None, "same", "")
previous_suggestions = [
Suggestion("link", "urlA"),
Suggestion("question", "question_data")
]
current_suggestions = [
Suggestion("link", "urlA"),
Suggestion("question", "question_data")
]
self.assertTrue(
analyse_same(request, current_suggestions, previous_suggestions))
def test_fail_when_expected_same_and_suggestions_are_different(self):
request = Request(None, None, "same", "")
previous_suggestions = [
Suggestion("link", "urlA"),
Suggestion("question", "question_data")
]
current_suggestions = [
Suggestion("link", "different_url"),
Suggestion("question", "question_data")
]
self.assertFalse(
analyse_same(request, current_suggestions, previous_suggestions))
def test_when_expecting_question_get_selected_suggestion_returns_first_question_and_position_one(
self):
suggestions = [
Suggestion("link", "some_dummy_url"),
Suggestion("link", "more_dummy_url"),
Suggestion("question", "question_data")
]
request = Request(None, None, "question", "")
selected_suggestion = get_selected_suggestion(suggestions, request)
self.assertIsNotNone(selected_suggestion)
self.assertEquals(1, selected_suggestion[0])
self.assertEquals(Suggestion("question", "question_data"),
selected_suggestion[1])
def test_when_expecting_link_get_selected_suggestion_returns_first_matching_link_and_position(
self):
suggestions = [
Suggestion("question", "dummy_question_data"),
Suggestion("link", "some_dummy_url"),
Suggestion("link", "expected_url")
]
request = Request(None, None, "link",
"some_url_not_found expected_url")
selected_suggestion = get_selected_suggestion(suggestions, request)
self.assertIsNotNone(selected_suggestion)
self.assertEquals(2, selected_suggestion[0])
self.assertEquals(Suggestion("link", "expected_url"),
selected_suggestion[1])
def test_when_expecting_notlink_get_selected_suggestion_returns_first_nonforbidden_link_and_position(
self):
suggestions = [
Suggestion("question", "dummy_question_data"),
Suggestion("link", "forbidden_url"),
Suggestion("link", "first_non_forbidden_url")
]
request = Request(None, None, "notlink",
"forbidden_url_not_in_suggestions forbidden_url")
selected_suggestion = get_selected_suggestion(suggestions, request)
self.assertIsNotNone(selected_suggestion)
self.assertEquals(2, selected_suggestion[0])
self.assertEquals(Suggestion("link", "first_non_forbidden_url"),
selected_suggestion[1])
def test_when_found_then_get_selected_suggestion_returns_matching_suggestion(
self):
suggestions = [
Suggestion("link", "some_dummy_url"),
Suggestion("link", "some_expected_url"),
Suggestion("link", "another_dummy_url")
]
request = Request(None, None, "link", "some_expected_url")
actual_selected_suggestion = get_selected_suggestion(
suggestions, request)
self.assertIsNotNone(actual_selected_suggestion)
self.assertEquals(2, actual_selected_suggestion[0])
self.assertEquals(Suggestion("link", "some_expected_url"),
actual_selected_suggestion[1])
def get_scenarios_from_csv_file(scenario_csv_file_path):
scenarios = []
with open(scenario_csv_file_path, newline='') as csv_file:
scenario_reader = csv.reader(csv_file, delimiter=',')
next(scenario_reader)
current_scenario_id = None
current_requests = []
for row in scenario_reader:
if row[0] != current_scenario_id:
update_scenarios(scenarios, current_requests)
current_scenario_id = row[0]
current_requests = []
request = Request(row[1], row[2], row[3], row[4])
current_requests.append(request)
update_scenarios(scenarios, current_requests)
return scenarios
def test_with_recommender_settings(self):
request = Request('asker', 'Hello world', 'To ignore', 'To ignore')
used_recommenders = [
RecommenderType.AnalyticsSearch, RecommenderType.FacetQuestion,
RecommenderType.NearestDocuments
]
actual_json = get_json_for_whisper_api('14', request, 'myChatKey',
used_recommenders)
expected_json = {
'chatkey': 'myChatKey',
'query': 'Hello world',
'type': 0,
'maxDocuments': 15,
'maxQuestions': 15,
'overridenRecommenderSettings': {
'useLongQuerySearchRecommender': False,
'usePreprocessedQuerySearchRecommender': False,
'useAnalyticsSearchRecommender': True,
'useFacetQuestionRecommender': True,
'useNearestDocumentsRecommender': True
}
}
self.assertEquals(expected_json, actual_json)
def test_average_selected_suggestion_position_only_counts_questions_when_expecting_questions(self):
suggestions = [
Suggestion('link', 'dummy_urlA'),
Suggestion('link', 'dummy_urlB'),
Suggestion('link', 'dummy_urlC'),
Suggestion('question', 'dummy_questionA'),
Suggestion('question', 'dummy_questionB'),
Suggestion('question', 'dummy_questionC')
]
request = Request(
None,
None,
"question",
""
)
scenarios = [Scenario([request])]
suggestions_responses_for_each_scenario = [[
SuggestionsResponse(
suggestions,
timedelta(seconds=42)
)
]]
metrics_analyzer = MetricsAnalyzer(scenarios, suggestions_responses_for_each_scenario)
self.assertEquals(1, metrics_analyzer.calculate_mean_position_of_selected_suggestions())