def test_build_suggest_query_all_logs_nonempty_stacktrace_launches_with_the_same_name(
self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
test_item_info = launch_objects.TestItemInfo(
**{
"analyzerConfig": {
"analyzerMode": "LAUNCH_NAME",
"numberOfLogLines": -1
},
"launchId": 12,
"launchName": "Launch name",
"project": 1,
"testCaseHash": 1,
"uniqueId": "unique",
"testItemId": 2
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "hello world 'sdf'",
"merged_small_logs": "",
"detected_message": "hello world 'sdf'",
"detected_message_with_numbers": "hello world 1 'sdf'",
"stacktrace": "invoke.method(arg)",
"only_numbers": "1",
"found_exceptions": "AssertionError",
"found_exceptions_extended": "AssertionError",
"message_params": "sdf",
"urls": "",
"paths": "",
"message_without_params_extended": "hello world",
"detected_message_without_params_extended": "hello world",
"stacktrace_extended": "invoke.method(arg)",
"message_extended": "hello world 'sdf'",
"detected_message_extended": "hello world 'sdf'",
"potential_status_codes": "200 401"
}
}
query_from_esclient = EsQueryBuilder(
search_cfg, 40000).build_suggest_query(
test_item_info,
log,
message_field="message_without_params_extended",
det_mes_field="detected_message_without_params_extended",
stacktrace_field="stacktrace_extended")
demo_query = utils.get_fixture(
self.
suggest_query_all_logs_nonempty_stacktrace_launches_with_the_same_name,
to_json=True)
query_from_esclient.should.equal(demo_query)
def test_build_suggest_query_merged_small_logs_search(self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
test_item_info = launch_objects.TestItemInfo(
**{
"analyzerConfig": {
"analyzerMode": "ALL",
"numberOfLogLines": -1
},
"launchId": 12,
"launchName": "Launch name",
"project": 1,
"testCaseHash": 1,
"uniqueId": "unique",
"testItemId": 2
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "",
"merged_small_logs": "hello world",
"detected_message": "",
"detected_message_with_numbers": "",
"stacktrace": "",
"only_numbers": "",
"found_exceptions": "AssertionError",
"found_exceptions_extended": "AssertionError",
"message_params": "",
"urls": "",
"paths": "",
"message_without_params_extended": "",
"detected_message_without_params_extended": "",
"stacktrace_extended": "",
"message_extended": "",
"detected_message_extended": "",
"potential_status_codes": "200 400"
}
}
query_from_service = SuggestService(
self.app_config, search_cfg).build_suggest_query(
test_item_info,
log,
message_field="message_extended",
det_mes_field="detected_message_extended",
stacktrace_field="stacktrace_extended")
demo_query = utils.get_fixture(
self.suggest_query_merged_small_logs_search, to_json=True)
query_from_service.should.equal(demo_query)
def test_suggest_items(self):
"""Test suggesting test items"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=1,
logs=[]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=2,
logs=[
launch_objects.Log(
logId=1,
message="error found",
logLevel=40000)
]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=1,
logs=[
launch_objects.Log(
logId=1,
message=" ",
logLevel=40000)
]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
}],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
}],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=80.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.2, 0.8]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.3, 0.7]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0], [[0.3, 0.7], [0.9, 0.1]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=45.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0], [[0.3, 0.7], [0.55, 0.45]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.three_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=3,
relevantLogId=3,
matchScore=80.0,
esScore=10.0,
esPosition=2,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=45.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=2,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0, 1], [[0.3, 0.7], [0.55, 0.45],
[0.2, 0.8]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_second),
"rs":
utils.get_fixture(
self.three_hits_search_rs_with_duplicate),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=3,
relevantLogId=3,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=70.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=2,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 1, 1], [[0.3, 0.7], [0.3, 0.7],
[0.3, 0.7]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_first),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_second),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_third),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_merged_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=-1,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=90.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.1, 0.9]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_first),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_second),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_third),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_merged_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
suggest_service = SuggestService(app_config=self.app_config,
search_cfg=config)
_boosting_decision_maker = BoostingDecisionMaker()
_boosting_decision_maker.get_feature_ids = MagicMock(
return_value=[0])
_boosting_decision_maker.predict = MagicMock(
return_value=test["boost_predict"])
suggest_service.suggest_decision_maker = _boosting_decision_maker
response = suggest_service.suggest_items(
test["test_item_info"])
response.should.have.length_of(len(test["expected_result"]))
for real_resp, expected_resp in zip(response,
test["expected_result"]):
real_resp.should.equal(expected_resp)
TestSuggestService.shutdown_server(test["test_calls"])
def prepare_test_item_info(test_item_info):
"""Function for deserializing test item info for suggestions"""
return launch_objects.TestItemInfo(**test_item_info)