def test_analyze_to_string(self):
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
self._scenarios, self._suggestions_responses_for_each_scenario)
expected_analysis_string = \
('Request,Scenario,Person,Message,Success condition,Result,System response time\n'
'0,1,agent,I love mushrooms,same,fail,0:00:49\n'
'1,2,asker,I have problems calling the rest API,link,success(1-1),0:00:49\n'
'2,2,asker,I have problems calling the rest API,link,success(1-1),0:00:49\n'
'3,2,agent,I love mushrooms,same,success,0:00:49\n'
'4,2,asker,I have problems calling the rest API,link,fail,0:00:54\n'
'5,2,asker,I have problems calling the rest API,link,fail,0:00:48\n'
'6,2,asker,Should fail notlink success condition,notlink,fail,0:00:48\n'
'7,2,asker,Should fail multiple unwanted links,notlink,fail,0:00:48\n'
'8,2,agent,Should succeed notlink success condition,notlink,success,0:00:48\n'
'9,2,agent,World,link,fail,0:00:48\n'
'10,2,agent,World,link,fail,0:00:48\n'
'11,3,asker,Hello,question,success(1-2),0:00:48\n'
'12,3,asker,Hello,question,success(1-2),0:00:48\n'
'13,3,asker,Hello,question,fail,0:00:48\n'
'14,3,asker,Hello,question,fail,0:00:48\n'
'15,3,asker,Hello,question,fail,0:00:48\n'
'16,3,asker,Hello,question,fail,0:00:48\n'
'17,3,asker,Hello,question,success,0:00:48\n'
'18,3,asker,Hello,question,fail,0:00:48\n'
'19,3,asker,Hello,link,success,0:00:48\n'
'20,3,asker,Hello,link,fail,0:00:48\n'
'\n8 of 21 tests passed')
analysis_string = suggestions_responses_analyzer.analyze_to_string()
self.assertEquals(expected_analysis_string, analysis_string)
def test_analyze_with_one_successful_suggestions_response(self):
expected_analysis = ['success(1-1)']
request = Request('asker', 'What is Coveo?', 'link',
'https://www.coveo.com/')
suggestion = Suggestion('link', 'https://www.coveo.com/')
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
[Scenario([request])],
[[SuggestionsResponse([suggestion], timedelta(seconds=0))]])
self.assertEquals(expected_analysis,
suggestions_responses_analyzer.analyze_scenarios())
def test_analyze(self):
expected_analysis = [
'fail', 'success(1-1)', 'success(1-1)', 'success', 'fail', 'fail',
'fail', 'fail', 'success', 'fail', 'fail', 'success(1-2)',
'success(1-2)', 'fail', 'fail', 'fail', 'fail', 'success', 'fail',
'success', 'fail'
]
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
self._scenarios, self._suggestions_responses_for_each_scenario)
self.assertEquals(expected_analysis,
suggestions_responses_analyzer.analyze_scenarios())
def test_same_at_start_of_scenario(self):
expected_analysis_string = \
('Request,Scenario,Person,Message,Success condition,Result,System response time\n'
'0,1,agent,link required, should pass,link,success(1-1),0:00:49\n'
'1,2,asker,same required, expects empty because start of scenario, should pass,same,success,0:00:49\n'
'2,2,agent,link required, should pass,link,success(1-1),0:00:49\n'
'3,3,asker,same required, expects empty because start of scenario, should fail,same,fail,0:00:49\n'
'\n3 of 4 tests passed')
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
self._scenarios_starting_with_same,
self._suggestions_responses_starting_with_same)
analysis_string = suggestions_responses_analyzer.analyze_to_string()
self.assertEquals(expected_analysis_string, analysis_string)
def test_constructor_with_no_suggestions_response_for_every_request(self):
with self.assertRaises(
NoSuggestionsResponsesForEveryRequestsException):
SuggestionsResponsesAnalyzer(
self._scenarios,
[SuggestionsResponse([], timedelta(seconds=10))])
with self.assertRaises(
NoSuggestionsResponsesForEveryRequestsException):
suggestions_responses_for_each_scenario_but_not_good_size = [
[SuggestionsResponse([], timedelta(seconds=10))],
[SuggestionsResponse([], timedelta(seconds=10))],
[SuggestionsResponse([], timedelta(seconds=10))],
]
SuggestionsResponsesAnalyzer(
self._scenarios,
suggestions_responses_for_each_scenario_but_not_good_size)
def run_whisper_automatic_test(whisper_api_base_url,
whisper_api_version,
is_verbose,
scenarios,
used_recommenders=None):
def get_suggestions(request, chatkey):
return get_suggestions_from_whisper_api(
whisper_api_version,
get_suggestions_endpoint(whisper_api_base_url), request, chatkey,
used_recommenders)
scenario_runner = ScenariosRunner(get_suggestions, get_time)
suggestions_responses = scenario_runner.run(scenarios)
suggestions_responses_analyzer = SuggestionsResponsesAnalyzer(
scenarios, suggestions_responses)
metrics_analyzer = MetricsAnalyzer(scenarios, suggestions_responses)
quality_indexes_analyzer = QualityIndexesAnalyzer(metrics_analyzer)
print_suggestions_responses_analysis(suggestions_responses_analyzer)
print_metrics(metrics_analyzer)
print_quality_indexes(quality_indexes_analyzer)
if is_verbose:
print_failing_requests_information(suggestions_responses_analyzer,
scenarios, suggestions_responses)
def test_constructor_with_no_suggestions_responses(self):
with self.assertRaises(NoSuggestionsResponsesException):
SuggestionsResponsesAnalyzer(self._scenarios, [])
def test_constructor_with_no_request_in_scenarios(self):
with self.assertRaises(NoRequestsException):
SuggestionsResponsesAnalyzer(
[Scenario([])], self._suggestions_responses_for_each_scenario)