def test_delete_index(self):
"""Test deleting an index"""
tests = [
{
"test_calls": [
{
"method": httpretty.DELETE,
"uri": "/1",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_deleted_rs),
},
],
"index":
1,
"result":
True,
},
{
"test_calls": [
{
"method": httpretty.DELETE,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_not_found_rs),
},
],
"index":
2,
"result":
False,
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.delete_index(test["index"])
test["result"].should.equal(response)
TestEsClient.shutdown_server(test["test_calls"])
def test_create_index(self):
"""Test creating index"""
tests = [
{
"test_calls": [
{
"method": httpretty.PUT,
"uri": "/idx0",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_created_rs),
},
],
"index":
"idx0",
"acknowledged":
True,
},
{
"test_calls": [
{
"method": httpretty.PUT,
"uri": "/idx1",
"status": HTTPStatus.BAD_REQUEST,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_already_exists_rs),
},
],
"index":
"idx1",
"acknowledged":
False,
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.create_index(test["index"])
response.acknowledged.should.equal(test["acknowledged"])
TestEsClient.shutdown_server(test["test_calls"])
def test_build_suggest_query_all_logs_nonempty_stacktrace_launches_with_the_same_name(
self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
test_item_info = launch_objects.TestItemInfo(
**{
"analyzerConfig": {
"analyzerMode": "LAUNCH_NAME",
"numberOfLogLines": -1
},
"launchId": 12,
"launchName": "Launch name",
"project": 1,
"testCaseHash": 1,
"uniqueId": "unique",
"testItemId": 2
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "hello world 'sdf'",
"merged_small_logs": "",
"detected_message": "hello world 'sdf'",
"detected_message_with_numbers": "hello world 1 'sdf'",
"stacktrace": "invoke.method(arg)",
"only_numbers": "1",
"found_exceptions": "AssertionError",
"found_exceptions_extended": "AssertionError",
"message_params": "sdf",
"urls": "",
"paths": "",
"message_without_params_extended": "hello world",
"detected_message_without_params_extended": "hello world",
"stacktrace_extended": "invoke.method(arg)",
"message_extended": "hello world 'sdf'",
"detected_message_extended": "hello world 'sdf'",
"potential_status_codes": "200 401"
}
}
query_from_esclient = EsQueryBuilder(
search_cfg, 40000).build_suggest_query(
test_item_info,
log,
message_field="message_without_params_extended",
det_mes_field="detected_message_without_params_extended",
stacktrace_field="stacktrace_extended")
demo_query = utils.get_fixture(
self.
suggest_query_all_logs_nonempty_stacktrace_launches_with_the_same_name,
to_json=True)
query_from_esclient.should.equal(demo_query)
def test_normalize_results(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [[{"_score": 158.08437,
"normalized_score": 1.0, }]],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [[{"_score": 158.08437,
"normalized_score": 1.0,
},
{"_score": 77.53298,
"normalized_score": 0.4904,
}, ]],
},
]
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"],
test["config"],
[],
weighted_log_similarity_calculator=weight_log_sim)
_boosting_featurizer.all_results.should.have.length_of(len(test["result"]))
for i in range(len(test["result"])):
for j in range(len(test["result"][i])):
for field in test["result"][i][j]:
elastic_res = _boosting_featurizer.all_results[i][1][j]
elastic_res[field].should.equal(test["result"][i][j][field],
epsilon=self.epsilon)
def test_build_suggest_query_merged_small_logs_search(self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
test_item_info = launch_objects.TestItemInfo(
**{
"analyzerConfig": {
"analyzerMode": "ALL",
"numberOfLogLines": -1
},
"launchId": 12,
"launchName": "Launch name",
"project": 1,
"testCaseHash": 1,
"uniqueId": "unique",
"testItemId": 2
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "",
"merged_small_logs": "hello world",
"detected_message": "",
"detected_message_with_numbers": "",
"stacktrace": "",
"only_numbers": "",
"found_exceptions": "AssertionError",
"found_exceptions_extended": "AssertionError",
"message_params": "",
"urls": "",
"paths": "",
"message_without_params_extended": "",
"detected_message_without_params_extended": "",
"stacktrace_extended": "",
"message_extended": "",
"detected_message_extended": "",
"potential_status_codes": "200 400"
}
}
query_from_service = SuggestService(
self.app_config, search_cfg).build_suggest_query(
test_item_info,
log,
message_field="message_extended",
det_mes_field="detected_message_extended",
stacktrace_field="stacktrace_extended")
demo_query = utils.get_fixture(
self.suggest_query_merged_small_logs_search, to_json=True)
query_from_service.should.equal(demo_query)
def test_list_indices(self):
"""Test checking getting indices from elasticsearch"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/_cat/indices?format=json",
"status": HTTPStatus.OK,
"rs": "[]",
},
],
"expected_count":
0,
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/_cat/indices?format=json",
"status": HTTPStatus.OK,
"rs": utils.get_fixture(self.two_indices_rs),
},
],
"expected_count":
2,
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/_cat/indices?format=json",
"status": HTTPStatus.INTERNAL_SERVER_ERROR,
},
],
"expected_count":
0,
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.list_indices()
response.should.have.length_of(test["expected_count"])
TestEsClient.shutdown_server(test["test_calls"])
def test_build_search_query(self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
search_req = launch_objects.SearchLogs(
**{
"launchId": 1,
"launchName": "launch 1",
"itemId": 2,
"projectId": 3,
"filteredLaunchIds": [1, 2, 3],
"logMessages": ["log message 1"],
"logLines": -1
})
query_from_esclient = EsQueryBuilder(search_cfg,
40000).build_search_query(
search_req, "log message 1")
demo_query = utils.get_fixture(self.query_search_logs, to_json=True)
query_from_esclient.should.equal(demo_query)
def test_build_analyze_query_all_logs_nonempty_stacktrace_launches_with_the_same_name(
self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
launch = launch_objects.Launch(
**{
"analyzerConfig": {
"analyzerMode": "LAUNCH_NAME",
"numberOfLogLines": -1
},
"launchId": 12,
"launchName": "Launch name",
"project": 1
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "hello world",
"merged_small_logs": "",
"detected_message": "hello world",
"detected_message_with_numbers": "hello world 1",
"stacktrace": "invoke.method(arg)",
"only_numbers": "1",
"found_exceptions": "AssertionError",
"potential_status_codes": "300 401"
}
}
query_from_esclient = EsQueryBuilder(search_cfg,
40000).build_analyze_query(
launch, log)
demo_query = utils.get_fixture(
self.
query_all_logs_nonempty_stacktrace_launches_with_the_same_name,
to_json=True)
query_from_esclient.should.equal(demo_query)
def test_build_analyze_query_two_log_lines_only_current_launch_wo_exceptions(
self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
launch = launch_objects.Launch(
**{
"analyzerConfig": {
"analyzerMode": "CURRENT_LAUNCH",
"numberOfLogLines": 2
},
"launchId": 12,
"launchName": "Launch name",
"project": 1
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "hello world",
"merged_small_logs": "",
"detected_message": "hello world",
"detected_message_with_numbers": "hello world 1",
"stacktrace": "",
"only_numbers": "1",
"found_exceptions": "",
"potential_status_codes": ""
}
}
query_from_esclient = EsQueryBuilder(search_cfg,
40000).build_analyze_query(
launch, log)
demo_query = utils.get_fixture(
self.query_two_log_lines_only_current_launch_wo_exceptions,
to_json=True)
query_from_esclient.should.equal(demo_query)
def test_build_analyze_query_all_logs_empty_stacktrace(self):
"""Tests building analyze query"""
search_cfg = TestEsQuery.get_default_search_config()
launch = launch_objects.Launch(
**{
"analyzerConfig": {
"analyzerMode": "ALL",
"numberOfLogLines": -1
},
"launchId": 12,
"launchName": "Launch name",
"project": 1
})
log = {
"_id": 1,
"_index": 1,
"_source": {
"unique_id": "unique",
"test_case_hash": 1,
"test_item": "123",
"message": "hello world",
"merged_small_logs": "",
"detected_message": "hello world",
"detected_message_with_numbers": "hello world 1",
"stacktrace": "",
"only_numbers": "1",
"found_exceptions": "AssertionError",
"potential_status_codes": ""
}
}
query_from_service = AutoAnalyzerService(
self.app_config, search_cfg).build_analyze_query(launch, log)
demo_query = utils.get_fixture(self.query_all_logs_empty_stacktrace,
to_json=True)
query_from_service.should.equal(demo_query)
def test_index_logs(self):
"""Test indexing logs from launches"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
"content_type": "application/json",
},
],
"index_rq":
utils.get_fixture(self.launch_wo_test_items),
"has_errors":
False,
"expected_count":
0,
"expected_log_exceptions": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
"content_type": "application/json",
},
],
"index_rq":
utils.get_fixture(self.launch_w_test_items_wo_logs),
"has_errors":
False,
"expected_count":
0,
"expected_log_exceptions": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
"content_type": "application/json",
},
],
"index_rq":
utils.get_fixture(self.launch_w_test_items_w_empty_logs),
"has_errors":
False,
"expected_count":
0,
"expected_log_exceptions": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
{
"method": httpretty.PUT,
"uri": "/2",
"status": HTTPStatus.OK,
"rs": utils.get_fixture(self.index_created_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq":
utils.get_fixture(self.index_logs_rq_big_messages),
"rs": utils.get_fixture(self.index_logs_rs),
},
{
"method":
httpretty.GET,
"uri":
"/2/_search?scroll=5m&size=1000",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_merged_logs),
"rs":
utils.get_fixture(
self.two_hits_search_with_big_messages_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method":
httpretty.GET,
"uri":
"/2/_search?scroll=5m&size=1000",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_not_merged_logs),
"rs":
utils.get_fixture(
self.two_hits_search_with_big_messages_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq":
utils.get_fixture(self.index_logs_rq_merged_logs),
"rs": utils.get_fixture(self.index_logs_rs),
},
],
"index_rq":
utils.get_fixture(self.launch_w_test_items_w_logs),
"has_errors":
False,
"expected_count":
2,
"expected_log_exceptions": [
launch_objects.LogExceptionResult(
logId=1,
foundExceptions=['java.lang.NoClassDefFoundError']),
launch_objects.LogExceptionResult(
logId=2,
foundExceptions=['java.lang.NoClassDefFoundError'])
]
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
{
"method": httpretty.PUT,
"uri": "/2",
"status": HTTPStatus.OK,
"rs": utils.get_fixture(self.index_created_rs),
},
{
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.index_logs_rq_different_log_level),
"rs":
utils.get_fixture(
self.index_logs_rs_different_log_level),
},
{
"method": httpretty.GET,
"uri": "/2/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method": httpretty.GET,
"uri": "/2/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_not_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.index_logs_rq_different_log_level_merged),
"rs":
utils.get_fixture(
self.index_logs_rs_different_log_level),
},
],
"index_rq":
utils.get_fixture(
self.launch_w_test_items_w_logs_different_log_level),
"has_errors":
False,
"expected_count":
1,
"expected_log_exceptions": [
launch_objects.LogExceptionResult(logId=1,
foundExceptions=[])
]
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
es_client.es_client.scroll = MagicMock(return_value=json.loads(
utils.get_fixture(self.no_hits_search_rs)))
launches = [
launch_objects.Launch(**launch)
for launch in json.loads(test["index_rq"])
]
response = es_client.index_logs(launches)
test["has_errors"].should.equal(response.errors)
test["expected_count"].should.equal(response.took)
test["expected_log_exceptions"].should.equal(
response.logResults)
TestEsClient.shutdown_server(test["test_calls"])
def test_suggest_items(self):
"""Test suggesting test items"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=1,
logs=[]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=2,
logs=[
launch_objects.Log(
logId=1,
message="error found",
logLevel=40000)
]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=1,
logs=[
launch_objects.Log(
logId=1,
message=" ",
logLevel=40000)
]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
}],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
}],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=80.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.2, 0.8]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.3, 0.7]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0], [[0.3, 0.7], [0.9, 0.1]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=45.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0], [[0.3, 0.7], [0.55, 0.45]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.three_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=3,
relevantLogId=3,
matchScore=80.0,
esScore=10.0,
esPosition=2,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=45.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=2,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0, 1], [[0.3, 0.7], [0.55, 0.45],
[0.2, 0.8]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_second),
"rs":
utils.get_fixture(
self.three_hits_search_rs_with_duplicate),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=3,
relevantLogId=3,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=70.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=2,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 1, 1], [[0.3, 0.7], [0.3, 0.7],
[0.3, 0.7]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_first),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_second),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_third),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_merged_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=-1,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=90.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.1, 0.9]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_first),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_second),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_third),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_merged_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
suggest_service = SuggestService(app_config=self.app_config,
search_cfg=config)
_boosting_decision_maker = BoostingDecisionMaker()
_boosting_decision_maker.get_feature_ids = MagicMock(
return_value=[0])
_boosting_decision_maker.predict = MagicMock(
return_value=test["boost_predict"])
suggest_service.suggest_decision_maker = _boosting_decision_maker
response = suggest_service.suggest_items(
test["test_item_info"])
response.should.have.length_of(len(test["expected_result"]))
for real_resp, expected_resp in zip(response,
test["expected_result"]):
real_resp.should.equal(expected_resp)
TestSuggestService.shutdown_server(test["test_calls"])
def test_filter_by_min_should_match_any(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(filter_fields=[],
filter_fields_any=[]),
"result": [],
},
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message"]),
"result": [],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(
self.one_hit_search_rs_explained_wo_params, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(
self.one_hit_search_rs_explained_wo_params, to_json=True))]
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(
self.one_hit_search_rs_explained_wo_params, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message"]),
"result": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))]
},
{
"elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True),
utils.get_fixture(self.one_hit_search_rs_small_logs, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": []
},
{
"elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))]
},
]
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = SuggestBoostingFeaturizer(
test["elastic_results"],
test["config"],
[],
weighted_log_similarity_calculator=weight_log_sim)
all_results = test["elastic_results"]
all_results = _boosting_featurizer.filter_by_min_should_match_any(
all_results,
fields=test["config"]["filter_min_should_match_any"])
all_results.should.have.length_of(len(test["result"]))
for idx, (log, hits) in enumerate(all_results):
log["_id"].should.equal(test["result"][idx][0]["_id"])
for i, hit in enumerate(hits["hits"]["hits"]):
hit["_id"].should.equal(test["result"][idx][1]["hits"]["hits"][i]["_id"])
def test_find_most_relevant_by_type_for_suggests(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {},
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {1: {"mrHit": {"_score": 158.08437,
"_id": "1"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 1.0, },
}
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {1: {"mrHit": {"_score": 158.08437,
"_id": "1"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 1.0, },
2: {"mrHit": {"_score": 77.53298,
"_id": "2"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 0.4905, },
}
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.three_hits_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {1: {"mrHit": {"_score": 158.08437,
"_id": "1"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 0.9392, },
2: {"mrHit": {"_score": 168.31,
"_id": "2"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 1.0, }
}
},
]
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = SuggestBoostingFeaturizer(
test["elastic_results"],
test["config"],
[],
weighted_log_similarity_calculator=weight_log_sim)
scores_by_issue_type = _boosting_featurizer.find_most_relevant_by_type()
scores_by_issue_type.should.have.length_of(len(test["result"]))
for issue_type in test["result"]:
scores_by_issue_type.keys().should.contain(issue_type)
elastic_res = scores_by_issue_type[issue_type]
for field in test["result"][issue_type]:
if type(test["result"][issue_type][field]) != dict:
elastic_res[field].should.equal(test["result"][issue_type][field],
epsilon=self.epsilon)
else:
for field_dict in test["result"][issue_type][field]:
result_field_dict = test["result"][issue_type][field][field_dict]
elastic_res[field][field_dict].should.equal(result_field_dict,
epsilon=self.epsilon)
def test_search_logs(self):
"""Test search logs"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_logs_rq),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=["error"],
logLines=-1),
"expected_count":
0
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=[""],
logLines=-1),
"expected_count":
0
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq),
"rs":
utils.get_fixture(self.one_hit_search_rs_search_logs),
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=["error"],
logLines=-1),
"expected_count":
0
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_not_found),
"rs":
utils.get_fixture(self.two_hits_search_rs_search_logs),
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=["error occured once"],
logLines=-1),
"expected_count":
1
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
search_service = SearchService(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = search_service.search_logs(test["rq"])
response.should.have.length_of(test["expected_count"])
TestSearchService.shutdown_server(test["test_calls"])
def test_find_clusters(self):
"""Test finding clusters"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**(utils.get_fixture(
self.launch_wo_test_items, to_json=True))[0]),
for_update=False,
numberOfLogLines=-1),
"expected_result": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**(utils.get_fixture(
self.launch_w_test_items_wo_logs, to_json=True))[0]),
for_update=False,
numberOfLogLines=-1),
"expected_result": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
},
],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**(utils.get_fixture(
self.launch_w_test_items_w_empty_logs, to_json=True)[0]
)),
for_update=False,
numberOfLogLines=-1),
"expected_result": []
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.cluster_update),
"rs": utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=False,
numberOfLogLines=-1),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.cluster_update_all_the_same),
"rs":
utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=False,
numberOfLogLines=2),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="1")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_first_group),
"rs":
utils.get_fixture(self.no_hits_search_rs),
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_second_group),
"rs":
utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.cluster_update),
"rs": utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=True,
numberOfLogLines=-1),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_first_group),
"rs":
utils.get_fixture(self.one_hit_search_rs_clustering),
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_second_group),
"rs":
utils.get_fixture(self.one_hit_search_rs_clustering),
}, {
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.cluster_update_es_update),
"rs":
utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=True,
numberOfLogLines=-1),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=111,
testItemId=12,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_first_group_2lines),
"rs":
utils.get_fixture(self.one_hit_search_rs_clustering),
}, {
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.cluster_update_all_the_same_es_update),
"rs":
utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=True,
numberOfLogLines=2),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=111,
testItemId=12,
project=2,
launchId=1,
clusterId="1")
]
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
_cluster_service = ClusterService(app_config=self.app_config,
search_cfg=config)
response = _cluster_service.find_clusters(test["launch_info"])
response.should.have.length_of(len(test["expected_result"]))
cluster_ids_dict = {}
for i in range(len(response)):
test["expected_result"][i].logId.should.equal(
response[i].logId)
if test["expected_result"][i].clusterId == "":
test["expected_result"][i].clusterId.should.equal(
response[i].clusterId)
elif test["expected_result"][
i].clusterId not in cluster_ids_dict:
cluster_ids_dict[test["expected_result"]
[i].clusterId] = response[i].clusterId
elif test["expected_result"][
i].clusterId in cluster_ids_dict:
expected_cluster_id = cluster_ids_dict[
test["expected_result"][i].clusterId]
expected_cluster_id.should.equal(response[i].clusterId)
for cluster_id in cluster_ids_dict:
test["test_calls"][-1]["rq"] = test["test_calls"][-1][
"rq"].replace(
"\"cluster_id\":\"%s\"" % cluster_id,
"\"cluster_id\":\"%s\"" %
cluster_ids_dict[cluster_id])
TestClusterService.shutdown_server(test["test_calls"])
def test_full_data_check_suggests(self):
print("Boost model folder suggests: ", self.suggest_boost_model_folder)
print("Weights model folder suggests: ", self.weights_folder)
decision_maker = BoostingDecisionMaker(
folder=self.suggest_boost_model_folder)
boost_model_results = utils.get_fixture(
self.suggest_boost_model_results, to_json=True)
tests = []
all_configs = [(-1, [
"detected_message_extended",
"detected_message_without_params_extended",
"detected_message_without_params_and_brackets"
], decision_maker),
(2, [
"message_extended",
"message_without_params_extended",
"message_without_params_and_brackets"
], decision_maker)]
for log_lines, filter_fields_any, _decision_maker in all_configs:
tests.extend([
{
"elastic_results":
[(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True)),
(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_only_small_logs,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
])
for idx, test in enumerate(tests):
feature_ids = test["decision_maker"].get_feature_ids()
weight_log_sim = None
if self.weights_folder.strip():
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
_boosting_featurizer = SuggestBoostingFeaturizer(
test["elastic_results"],
test["config"],
feature_ids,
weighted_log_similarity_calculator=weight_log_sim)
with sure.ensure('Error in the test case number: {0}', idx):
gathered_data, test_item_ids = _boosting_featurizer.gather_features_info(
)
predict_label, predict_probability = test[
"decision_maker"].predict(gathered_data)
gathered_data.should.equal(boost_model_results[str(idx)][0],
epsilon=self.epsilon)
predict_label.tolist().should.equal(
boost_model_results[str(idx)][1], epsilon=self.epsilon)
predict_probability.tolist().should.equal(
boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_clean_index(self):
"""Test cleaning index logs"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search?scroll=5m&size=1000",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.search_not_merged_logs_for_delete),
"rs":
utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_not_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.index_logs_rq),
"rs": utils.get_fixture(self.index_logs_rs),
},
],
"rq":
launch_objects.CleanIndex(ids=[1], project=1),
"expected_count":
1
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
],
"rq":
launch_objects.CleanIndex(ids=[1], project=2),
"expected_count":
0
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
es_client.es_client.scroll = MagicMock(return_value=json.loads(
utils.get_fixture(self.no_hits_search_rs)))
response = es_client.delete_logs(test["rq"])
test["expected_count"].should.equal(response)
TestEsClient.shutdown_server(test["test_calls"])
def test_full_data_check(self):
print("Boost model folder : ", self.boost_model_folder)
print("Weights model folder : ", self.weights_folder)
print("Global defect type model folder : ",
self.global_defect_type_model_folder)
decision_maker = BoostingDecisionMaker(folder=self.boost_model_folder)
boost_model_results = utils.get_fixture(self.boost_model_results,
to_json=True)
tests = []
for log_lines, filter_fields, _decision_maker in [
(-1, ["detected_message", "stacktrace"], decision_maker),
(2, ["message"], decision_maker)
]:
tests.extend([
{
"elastic_results":
[(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_only_small_logs,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
])
for idx, test in enumerate(tests):
feature_ids = test["decision_maker"].get_feature_ids()
weight_log_sim = None
if self.weights_folder.strip():
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"],
test["config"],
feature_ids,
weighted_log_similarity_calculator=weight_log_sim)
if self.global_defect_type_model_folder.strip():
_boosting_featurizer.set_defect_type_model(
defect_type_model.DefectTypeModel(
folder=self.global_defect_type_model_folder))
with sure.ensure('Error in the test case number: {0}', idx):
gathered_data, issue_type_names = _boosting_featurizer.gather_features_info(
)
gathered_data.should.equal(boost_model_results[str(idx)][0],
epsilon=self.epsilon)
predict_label, predict_probability = test[
"decision_maker"].predict(gathered_data)
predict_label.tolist().should.equal(
boost_model_results[str(idx)][1], epsilon=self.epsilon)
predict_probability.tolist().should.equal(
boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_analyze_logs(self):
"""Test analyzing logs"""
tests = [
{
"test_calls": [{"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, ],
"index_rq": utils.get_fixture(self.launch_wo_test_items),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, ],
"index_rq": utils.get_fixture(
self.launch_w_test_items_wo_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, ],
"index_rq": utils.get_fixture(
self.launch_w_test_items_w_empty_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, ],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.no_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
}, ],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.one_hit_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1], [[0.2, 0.8]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.one_hit_search_rs, to_json=True),
utils.get_fixture(self.two_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1, 1], [[0.2, 0.8], [0.3, 0.7]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "PB001",
"boost_predict": ([0, 1], [[0.8, 0.2], [0.3, 0.7]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(
self.launch_w_test_items_w_logs_to_be_merged),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [
utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs_with_one_unique_id, to_json=True)],
"index_rq": utils.get_fixture(
self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1], [[0.2, 0.8]])
}
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
analyzer_service = AutoAnalyzerService(app_config=self.app_config,
search_cfg=config)
_boosting_decision_maker = BoostingDecisionMaker()
_boosting_decision_maker.get_feature_ids = MagicMock(return_value=[0])
_boosting_decision_maker.predict = MagicMock(return_value=test["boost_predict"])
if "msearch_results" in test:
analyzer_service.es_client.es_client.msearch = MagicMock(
return_value={"responses": test["msearch_results"]})
analyzer_service.boosting_decision_maker = _boosting_decision_maker
launches = [launch_objects.Launch(**launch)
for launch in json.loads(test["index_rq"])]
response = analyzer_service.analyze_logs(launches)
response.should.have.length_of(test["expected_count"])
if test["expected_issue_type"] != "":
test["expected_issue_type"].should.equal(response[0].issueType)
if "expected_id" in test:
test["expected_id"].should.equal(response[0].relevantItem)
TestAutoAnalyzerService.shutdown_server(test["test_calls"])
