def test_build_analyze_query_all_logs_nonempty_stacktrace_launches_with_the_same_name( self): """Tests building analyze query""" search_cfg = TestEsQuery.get_default_search_config() launch = launch_objects.Launch( **{ "analyzerConfig": { "analyzerMode": "LAUNCH_NAME", "numberOfLogLines": -1 }, "launchId": 12, "launchName": "Launch name", "project": 1 }) log = { "_id": 1, "_index": 1, "_source": { "unique_id": "unique", "test_case_hash": 1, "test_item": "123", "message": "hello world", "merged_small_logs": "", "detected_message": "hello world", "detected_message_with_numbers": "hello world 1", "stacktrace": "invoke.method(arg)", "only_numbers": "1", "found_exceptions": "AssertionError", "potential_status_codes": "300 401" } } query_from_esclient = EsQueryBuilder(search_cfg, 40000).build_analyze_query( launch, log) demo_query = utils.get_fixture( self. query_all_logs_nonempty_stacktrace_launches_with_the_same_name, to_json=True) query_from_esclient.should.equal(demo_query)
def test_build_analyze_query_two_log_lines_only_current_launch_wo_exceptions( self): """Tests building analyze query""" search_cfg = TestEsQuery.get_default_search_config() launch = launch_objects.Launch( **{ "analyzerConfig": { "analyzerMode": "CURRENT_LAUNCH", "numberOfLogLines": 2 }, "launchId": 12, "launchName": "Launch name", "project": 1 }) log = { "_id": 1, "_index": 1, "_source": { "unique_id": "unique", "test_case_hash": 1, "test_item": "123", "message": "hello world", "merged_small_logs": "", "detected_message": "hello world", "detected_message_with_numbers": "hello world 1", "stacktrace": "", "only_numbers": "1", "found_exceptions": "", "potential_status_codes": "" } } query_from_esclient = EsQueryBuilder(search_cfg, 40000).build_analyze_query( launch, log) demo_query = utils.get_fixture( self.query_two_log_lines_only_current_launch_wo_exceptions, to_json=True) query_from_esclient.should.equal(demo_query)
def test_build_analyze_query_all_logs_empty_stacktrace(self): """Tests building analyze query""" search_cfg = TestEsQuery.get_default_search_config() launch = launch_objects.Launch( **{ "analyzerConfig": { "analyzerMode": "ALL", "numberOfLogLines": -1 }, "launchId": 12, "launchName": "Launch name", "project": 1 }) log = { "_id": 1, "_index": 1, "_source": { "unique_id": "unique", "test_case_hash": 1, "test_item": "123", "message": "hello world", "merged_small_logs": "", "detected_message": "hello world", "detected_message_with_numbers": "hello world 1", "stacktrace": "", "only_numbers": "1", "found_exceptions": "AssertionError", "potential_status_codes": "" } } query_from_service = AutoAnalyzerService( self.app_config, search_cfg).build_analyze_query(launch, log) demo_query = utils.get_fixture(self.query_all_logs_empty_stacktrace, to_json=True) query_from_service.should.equal(demo_query)
def test_build_analyze_query(self): """Tests building analyze query""" search_cfg = { "MinShouldMatch": "80%", "BoostAA": 10, "BoostLaunch": 5, "BoostUniqueID": 3, "MaxQueryTerms": 50, } error_logging_level = 40000 es_client = esclient.EsClient(search_cfg=search_cfg) launch = launch_objects.Launch(**{ "analyzerConfig": {"analyzerMode": "SearchModeAll"}, "launchId": 123, "launchName": "Launch name", "project": 1}) log = { "_id": 1, "_index": 1, "_source": { "unique_id": "unique", "test_case_hash": 1, "test_item": 123, "message": "hello world", "merged_small_logs": "", "detected_message": "hello world", "detected_message_with_numbers": "hello world 1", "stacktrace": "", "only_numbers": "1"}} query_from_esclient = es_client.build_analyze_query(launch, "unique", log) demo_query = TestEsQuery.build_demo_query(search_cfg, "Launch name", "unique", log, error_logging_level) query_from_esclient.should.equal(demo_query)
def prepare_launches(launches): """Function for deserializing array of launches""" return [launch_objects.Launch(**launch) for launch in launches]
def test_index_logs(self): """Test indexing logs from launches""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, "content_type": "application/json", }, ], "index_rq": utils.get_fixture(self.launch_wo_test_items), "has_errors": False, "expected_count": 0, "expected_log_exceptions": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, "content_type": "application/json", }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_wo_logs), "has_errors": False, "expected_count": 0, "expected_log_exceptions": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, "content_type": "application/json", }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_w_empty_logs), "has_errors": False, "expected_count": 0, "expected_log_exceptions": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, { "method": httpretty.PUT, "uri": "/2", "status": HTTPStatus.OK, "rs": utils.get_fixture(self.index_created_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.index_logs_rq_big_messages), "rs": utils.get_fixture(self.index_logs_rs), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_merged_logs), "rs": utils.get_fixture( self.two_hits_search_with_big_messages_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.delete_logs_rs), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_not_merged_logs), "rs": utils.get_fixture( self.two_hits_search_with_big_messages_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.index_logs_rq_merged_logs), "rs": utils.get_fixture(self.index_logs_rs), }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "has_errors": False, "expected_count": 2, "expected_log_exceptions": [ launch_objects.LogExceptionResult( logId=1, foundExceptions=['java.lang.NoClassDefFoundError']), launch_objects.LogExceptionResult( logId=2, foundExceptions=['java.lang.NoClassDefFoundError']) ] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, { "method": httpretty.PUT, "uri": "/2", "status": HTTPStatus.OK, "rs": utils.get_fixture(self.index_created_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.index_logs_rq_different_log_level), "rs": utils.get_fixture( self.index_logs_rs_different_log_level), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_merged_logs), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.delete_logs_rs), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_not_merged_logs), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.index_logs_rq_different_log_level_merged), "rs": utils.get_fixture( self.index_logs_rs_different_log_level), }, ], "index_rq": utils.get_fixture( self.launch_w_test_items_w_logs_different_log_level), "has_errors": False, "expected_count": 1, "expected_log_exceptions": [ launch_objects.LogExceptionResult(logId=1, foundExceptions=[]) ] }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) es_client.es_client.scroll = MagicMock(return_value=json.loads( utils.get_fixture(self.no_hits_search_rs))) launches = [ launch_objects.Launch(**launch) for launch in json.loads(test["index_rq"]) ] response = es_client.index_logs(launches) test["has_errors"].should.equal(response.errors) test["expected_count"].should.equal(response.took) test["expected_log_exceptions"].should.equal( response.logResults) TestEsClient.shutdown_server(test["test_calls"])
def test_find_clusters(self): """Test finding clusters""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**(utils.get_fixture( self.launch_wo_test_items, to_json=True))[0]), for_update=False, numberOfLogLines=-1), "expected_result": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**(utils.get_fixture( self.launch_w_test_items_wo_logs, to_json=True))[0]), for_update=False, numberOfLogLines=-1), "expected_result": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, ], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**(utils.get_fixture( self.launch_w_test_items_w_empty_logs, to_json=True)[0] )), for_update=False, numberOfLogLines=-1), "expected_result": [] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=False, numberOfLogLines=-1), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update_all_the_same), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=False, numberOfLogLines=2), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="1") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_first_group), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_second_group), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=True, numberOfLogLines=-1), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_first_group), "rs": utils.get_fixture(self.one_hit_search_rs_clustering), }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_second_group), "rs": utils.get_fixture(self.one_hit_search_rs_clustering), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update_es_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=True, numberOfLogLines=-1), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=111, testItemId=12, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_first_group_2lines), "rs": utils.get_fixture(self.one_hit_search_rs_clustering), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.cluster_update_all_the_same_es_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=True, numberOfLogLines=2), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=111, testItemId=12, project=2, launchId=1, clusterId="1") ] }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) config = self.get_default_search_config() _cluster_service = ClusterService(app_config=self.app_config, search_cfg=config) response = _cluster_service.find_clusters(test["launch_info"]) response.should.have.length_of(len(test["expected_result"])) cluster_ids_dict = {} for i in range(len(response)): test["expected_result"][i].logId.should.equal( response[i].logId) if test["expected_result"][i].clusterId == "": test["expected_result"][i].clusterId.should.equal( response[i].clusterId) elif test["expected_result"][ i].clusterId not in cluster_ids_dict: cluster_ids_dict[test["expected_result"] [i].clusterId] = response[i].clusterId elif test["expected_result"][ i].clusterId in cluster_ids_dict: expected_cluster_id = cluster_ids_dict[ test["expected_result"][i].clusterId] expected_cluster_id.should.equal(response[i].clusterId) for cluster_id in cluster_ids_dict: test["test_calls"][-1]["rq"] = test["test_calls"][-1][ "rq"].replace( "\"cluster_id\":\"%s\"" % cluster_id, "\"cluster_id\":\"%s\"" % cluster_ids_dict[cluster_id]) TestClusterService.shutdown_server(test["test_calls"])
def test_analyze_logs(self): """Test analyzing logs""" tests = [ { "test_calls": [{"method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "index_rq": utils.get_fixture(self.launch_wo_test_items), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "index_rq": utils.get_fixture( self.launch_w_test_items_wo_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, ], "index_rq": utils.get_fixture( self.launch_w_test_items_w_empty_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, ], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.no_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.one_hit_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1], [[0.2, 0.8]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.one_hit_search_rs, to_json=True), utils.get_fixture(self.two_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1, 1], [[0.2, 0.8], [0.3, 0.7]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "PB001", "boost_predict": ([0, 1], [[0.8, 0.2], [0.3, 0.7]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture( self.launch_w_test_items_w_logs_to_be_merged), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [ utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs_with_one_unique_id, to_json=True)], "index_rq": utils.get_fixture( self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1], [[0.2, 0.8]]) } ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) config = self.get_default_search_config() analyzer_service = AutoAnalyzerService(app_config=self.app_config, search_cfg=config) _boosting_decision_maker = BoostingDecisionMaker() _boosting_decision_maker.get_feature_ids = MagicMock(return_value=[0]) _boosting_decision_maker.predict = MagicMock(return_value=test["boost_predict"]) if "msearch_results" in test: analyzer_service.es_client.es_client.msearch = MagicMock( return_value={"responses": test["msearch_results"]}) analyzer_service.boosting_decision_maker = _boosting_decision_maker launches = [launch_objects.Launch(**launch) for launch in json.loads(test["index_rq"])] response = analyzer_service.analyze_logs(launches) response.should.have.length_of(test["expected_count"]) if test["expected_issue_type"] != "": test["expected_issue_type"].should.equal(response[0].issueType) if "expected_id" in test: test["expected_id"].should.equal(response[0].relevantItem) TestAutoAnalyzerService.shutdown_server(test["test_calls"])