def test_number_of_suggested_questions_when_not_all_questions_updated(self):
suggestions_responses_for_each_scenario = [
[
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=49)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=49))
],
[
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=54)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48))
]
]
metrics_analyzer = MetricsAnalyzer(self._scenarios, suggestions_responses_for_each_scenario)
self.assertEquals(6, metrics_analyzer.calculate_number_of_suggested_questions())
def test_average_selected_suggestion_position_only_counts_questions_when_expecting_questions(self):
suggestions = [
Suggestion('link', 'dummy_urlA'),
Suggestion('link', 'dummy_urlB'),
Suggestion('link', 'dummy_urlC'),
Suggestion('question', 'dummy_questionA'),
Suggestion('question', 'dummy_questionB'),
Suggestion('question', 'dummy_questionC')
]
request = Request(
None,
None,
"question",
""
)
scenarios = [Scenario([request])]
suggestions_responses_for_each_scenario = [[
SuggestionsResponse(
suggestions,
timedelta(seconds=42)
)
]]
metrics_analyzer = MetricsAnalyzer(scenarios, suggestions_responses_for_each_scenario)
self.assertEquals(1, metrics_analyzer.calculate_mean_position_of_selected_suggestions())
def test_constructor_with_invalid_response_timestamp(self):
invalid_timestamp_suggestions_responses_for_each_scenario = [
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
],
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=-666)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
]
]
with self.assertRaises(InvalidTimestampException):
MetricsAnalyzer(self._scenarios, invalid_timestamp_suggestions_responses_for_each_scenario)
def run_whisper_automatic_test(whisper_api_base_url,
whisper_api_version,
is_verbose,
scenarios,
used_recommenders=None):
def get_suggestions(request, chatkey):
return get_suggestions_from_whisper_api(
whisper_api_version,
get_suggestions_endpoint(whisper_api_base_url), request, chatkey,
used_recommenders)
scenario_runner = ScenariosRunner(get_suggestions, get_time)
suggestions_responses = scenario_runner.run(scenarios)
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
scenarios, suggestions_responses)
metrics_analyzer = MetricsAnalyzer(scenarios, suggestions_responses)
quality_indexes_analyzer = QualityIndexesAnalyzer(metrics_analyzer)
print_suggestions_responses_analysis(suggestions_responses_analyzer)
print_metrics(metrics_analyzer)
print_quality_indexes(quality_indexes_analyzer)
if is_verbose:
print_failing_requests_information(suggestions_responses_analyzer,
scenarios, suggestions_responses)
def setUp(self):
self._suggestions_3_links = [
Suggestion('link', 'https://blog.coveo.com/goodbye-gsa-hello-intelligent-search-in-the-cloud/'),
Suggestion('link', 'https://coveo.github.io/search-ui/interfaces/isearchendpointoptions.html'),
Suggestion('link', 'https://onlinehelp.coveo.com/en/ces/7.0/administrator/about_net_conversion_scripts.htm')
]
self._suggestions_3_questions = [
Suggestion('question', 'What is your name?'),
Suggestion('question', 'Did you try this?'),
Suggestion('question', 'Hello?')
]
suggestions_responses_for_each_scenario = [
[
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=49)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=49))
],
[
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=54)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48))
]
]
self._scenarios = get_scenarios_from_csv_file(SCENARIO_FILE_PATH)
self._metrics_analyzer = MetricsAnalyzer(self._scenarios, suggestions_responses_for_each_scenario)
empty_suggestions_suggestions_responses_for_each_scenario = [
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
],
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
]
]
self._no_suggestions_responses_metrics_analyzer = \
MetricsAnalyzer(self._scenarios, empty_suggestions_suggestions_responses_for_each_scenario)
self._immediate_response_metrics_analyzer = \
MetricsAnalyzer(self._scenarios, empty_suggestions_suggestions_responses_for_each_scenario)
def test_constructor_with_no_suggestions_response_for_every_request(self):
with self.assertRaises(NoSuggestionsResponsesForEveryRequestsException):
MetricsAnalyzer(self._scenarios, [[SuggestionsResponse([], timedelta(seconds=10))]])
def test_constructor_with_no_suggestions_responses(self):
with self.assertRaises(NoSuggestionsResponsesException):
MetricsAnalyzer(self._scenarios, [])
def test_constructor_with_no_request_in_scenarios(self):
with self.assertRaises(NoRequestsException):
MetricsAnalyzer([Scenario([])], [[SuggestionsResponse([], timedelta(seconds=10))]])
class TestMetricsAnalyzer(unittest.TestCase):
def setUp(self):
self._suggestions_3_links = [
Suggestion('link', 'https://blog.coveo.com/goodbye-gsa-hello-intelligent-search-in-the-cloud/'),
Suggestion('link', 'https://coveo.github.io/search-ui/interfaces/isearchendpointoptions.html'),
Suggestion('link', 'https://onlinehelp.coveo.com/en/ces/7.0/administrator/about_net_conversion_scripts.htm')
]
self._suggestions_3_questions = [
Suggestion('question', 'What is your name?'),
Suggestion('question', 'Did you try this?'),
Suggestion('question', 'Hello?')
]
suggestions_responses_for_each_scenario = [
[
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=49)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=49))
],
[
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=54)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48))
]
]
self._scenarios = get_scenarios_from_csv_file(SCENARIO_FILE_PATH)
self._metrics_analyzer = MetricsAnalyzer(self._scenarios, suggestions_responses_for_each_scenario)
empty_suggestions_suggestions_responses_for_each_scenario = [
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
],
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
]
]
self._no_suggestions_responses_metrics_analyzer = \
MetricsAnalyzer(self._scenarios, empty_suggestions_suggestions_responses_for_each_scenario)
self._immediate_response_metrics_analyzer = \
MetricsAnalyzer(self._scenarios, empty_suggestions_suggestions_responses_for_each_scenario)
def test_constructor_with_no_request_in_scenarios(self):
with self.assertRaises(NoRequestsException):
MetricsAnalyzer([Scenario([])], [[SuggestionsResponse([], timedelta(seconds=10))]])
def test_constructor_with_no_suggestions_responses(self):
with self.assertRaises(NoSuggestionsResponsesException):
MetricsAnalyzer(self._scenarios, [])
def test_constructor_with_invalid_response_timestamp(self):
invalid_timestamp_suggestions_responses_for_each_scenario = [
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
],
[
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=-666)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0)),
SuggestionsResponse([], timedelta(seconds=0))
]
]
with self.assertRaises(InvalidTimestampException):
MetricsAnalyzer(self._scenarios, invalid_timestamp_suggestions_responses_for_each_scenario)
def test_constructor_with_no_suggestions_response_for_every_request(self):
with self.assertRaises(NoSuggestionsResponsesForEveryRequestsException):
MetricsAnalyzer(self._scenarios, [[SuggestionsResponse([], timedelta(seconds=10))]])
def test_average_system_response_time(self):
actual = self._metrics_analyzer.calculate_average_system_response_time()
expected = timedelta(seconds=49.143)
self.assertAlmostEquals(expected, actual, delta=timedelta(seconds=0.001))
def test_average_system_response_time_with_immediate_response(self):
self.assertAlmostEquals(
timedelta(seconds=0),
self._immediate_response_metrics_analyzer.calculate_average_system_response_time()
)
def test_messages_number(self):
self.assertEquals(7, self._metrics_analyzer.calculate_messages_number())
def test_average_selected_suggestion_position(self):
self.assertEquals(1, self._metrics_analyzer.calculate_mean_position_of_selected_suggestions())
def test_average_selected_suggestion_position_when_no_suggestion_is_selected(self):
self.assertEquals(
math.inf,
self._no_suggestions_responses_metrics_analyzer.calculate_mean_position_of_selected_suggestions()
)
def test_average_selected_suggestion_position_only_counts_questions_when_expecting_questions(self):
suggestions = [
Suggestion('link', 'dummy_urlA'),
Suggestion('link', 'dummy_urlB'),
Suggestion('link', 'dummy_urlC'),
Suggestion('question', 'dummy_questionA'),
Suggestion('question', 'dummy_questionB'),
Suggestion('question', 'dummy_questionC')
]
request = Request(
None,
None,
"question",
""
)
scenarios = [Scenario([request])]
suggestions_responses_for_each_scenario = [[
SuggestionsResponse(
suggestions,
timedelta(seconds=42)
)
]]
metrics_analyzer = MetricsAnalyzer(scenarios, suggestions_responses_for_each_scenario)
self.assertEquals(1, metrics_analyzer.calculate_mean_position_of_selected_suggestions())
def test_total_number_of_suggestions_updates(self):
self.assertEquals(4, self._metrics_analyzer.calculate_total_number_of_suggestions_updates())
def test_total_number_of_suggestions_updates_with_no_suggestions_updates(self):
self.assertEquals(
0,
self._no_suggestions_responses_metrics_analyzer.calculate_total_number_of_suggestions_updates()
)
def test_number_of_unwanted_suggestions_updates(self):
self.assertEquals(
2,
self._metrics_analyzer.calculate_number_of_unwanted_suggestions_updates()
)
def test_number_of_unwanted_suggestions_updates_when_no_suggestions_update(self):
self.assertEquals(
0,
self._no_suggestions_responses_metrics_analyzer.calculate_number_of_unwanted_suggestions_updates()
)
def test_number_of_selected_suggestions(self):
self.assertEquals(2, self._metrics_analyzer.calculate_number_of_selected_suggestions())
def test_number_of_selected_suggestions_when_there_are_none(self):
self.assertEquals(0, self._no_suggestions_responses_metrics_analyzer.calculate_number_of_selected_suggestions())
def test_number_of_suggested_questions(self):
self.assertEquals(6, self._metrics_analyzer.calculate_number_of_suggested_questions())
def test_number_of_suggested_questions_with_no_suggestions_responses(self):
self.assertEquals(0, self._no_suggestions_responses_metrics_analyzer.calculate_number_of_suggested_questions())
def test_number_of_suggested_questions_when_not_all_questions_updated(self):
suggestions_responses_for_each_scenario = [
[
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=49)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=49))
],
[
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=54)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_links, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48)),
SuggestionsResponse(self._suggestions_3_questions, timedelta(seconds=48))
]
]
metrics_analyzer = MetricsAnalyzer(self._scenarios, suggestions_responses_for_each_scenario)
self.assertEquals(6, metrics_analyzer.calculate_number_of_suggested_questions())
def test_number_of_suggested_links(self):
self.assertEquals(6, self._metrics_analyzer.calculate_number_of_suggested_links())
def test_mean_confidence_level_of_selected_suggestions(self):
self.assertEquals(0, self._metrics_analyzer.calculate_mean_confidence_level_of_selected_suggestions())