def test_create_custom_lambda_resource(): ######### # Integration test using a Custom Resource # Create a Lambda # CF will call the Lambda # The Lambda should call CF, to indicate success (using the cfnresponse-module) # This HTTP request will include any outputs that are now stored against the stack # TEST: verify that this output is persisted ########## if not settings.TEST_SERVER_MODE: raise SkipTest( "Needs a standalone MotoServer, as cfnresponse needs to connect to something" ) # Create cloudformation stack stack_name = f"stack{str(uuid4())[0:6]}" template_body = get_template(get_lambda_code()) cf = boto3.client("cloudformation", region_name="us-east-1") cf.create_stack( StackName=stack_name, TemplateBody=json.dumps(template_body), Capabilities=["CAPABILITY_IAM"], ) # Verify CloudWatch contains the correct logs log_group_name = get_log_group_name(cf, stack_name) success, logs = wait_for_log_msg(expected_msg="Status code: 200", log_group=log_group_name) with sure.ensure(f"Logs should indicate success: \n{logs}"): success.should.equal(True) # Verify the correct Output was returned outputs = get_outputs(cf, stack_name) outputs.should.have.length_of(1) outputs[0].should.have.key("OutputKey").equals("infokey") outputs[0].should.have.key("OutputValue").equals("special value")
def video(self, allow_redirects=False, size_limit=None): r = self.obj.should.be.downloadable(allow_redirects=allow_redirects) if size_limit: r.headers.should.have.key('Content-Length') source_size = int(r.headers['Content-Length']) size_limit = int(size_limit) with ensure( "source files size ({}) should be less limit size ({})".format( humanize.naturalsize(source_size), humanize.naturalsize(size_limit))): source_size.should.be.lower_than(size_limit) return r
def test_list_indices(self): """Test checking getting indices from elasticsearch""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/_cat/indices?format=json", "status": HTTPStatus.OK, "rs": "[]", }, ], "expected_count": 0, }, { "test_calls": [ { "method": httpretty.GET, "uri": "/_cat/indices?format=json", "status": HTTPStatus.OK, "rs": utils.get_fixture(self.two_indices_rs), }, ], "expected_count": 2, }, { "test_calls": [ { "method": httpretty.GET, "uri": "/_cat/indices?format=json", "status": HTTPStatus.INTERNAL_SERVER_ERROR, }, ], "expected_count": 0, }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) response = es_client.list_indices() response.should.have.length_of(test["expected_count"]) TestEsClient.shutdown_server(test["test_calls"])
def test_full_data_check(self): print("Boost model folder: ", self.boost_model_folder) decision_maker = BoostingDecisionMaker(self.boost_model_folder) boost_model_results = self.get_fixture(self.boost_model_results) tests = [ { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.one_hit_search_rs_explained))], "config": TestBoostingModel.get_default_config(), }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.two_hits_search_rs_explained))], "config": TestBoostingModel.get_default_config(), }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.two_hits_search_rs_explained)), (self.get_fixture(self.log_message), self.get_fixture(self.one_hit_search_rs_explained))], "config": TestBoostingModel.get_default_config(), }, { "elastic_results": [(self.get_fixture(self.log_message_only_small_logs), self.get_fixture(self.two_hits_search_rs_small_logs))], "config": TestBoostingModel.get_default_config(), }, ] for idx, test in enumerate(tests): _boosting_featurizer = BoostingFeaturizer( test["elastic_results"], test["config"], decision_maker.get_feature_ids()) with sure.ensure('Error in the test case number: {0}', idx): gathered_data, issue_type_names = _boosting_featurizer.gather_features_info( ) gathered_data.should.equal(boost_model_results[str(idx)][0], epsilon=self.epsilon) predict_label, predict_probability = decision_maker.predict( gathered_data) predict_label.tolist().should.equal( boost_model_results[str(idx)][1], epsilon=self.epsilon) predict_probability.tolist().should.equal( boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_normalize_results(self): tests = [ { "elastic_results": [], "config": TestBoostingFeaturizer.get_default_config(), "result": [], }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.one_hit_search_rs_explained))], "config": TestBoostingFeaturizer.get_default_config(), "result": [[{ "_score": 158.08437, "normalized_score": 1.0, }]], }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.two_hits_search_rs_explained))], "config": TestBoostingFeaturizer.get_default_config(), "result": [[ { "_score": 158.08437, "normalized_score": 1.0, }, { "_score": 77.53298, "normalized_score": 0.4904, }, ]], }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): _boosting_featurizer = BoostingFeaturizer( test["elastic_results"], test["config"], []) _boosting_featurizer.all_results.should.have.length_of( len(test["result"])) for i in range(len(test["result"])): for j in range(len(test["result"][i])): for field in test["result"][i][j]: elastic_res = _boosting_featurizer.all_results[i][ 1][j] elastic_res[field].should.equal( test["result"][i][j][field], epsilon=self.epsilon)
def test_delete_index(self): """Test deleting an index""" tests = [ { "test_calls": [ { "method": httpretty.DELETE, "uri": "/1", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.index_deleted_rs), }, ], "index": 1, "result": True, }, { "test_calls": [ { "method": httpretty.DELETE, "uri": "/2", "status": HTTPStatus.NOT_FOUND, "content_type": "application/json", "rs": utils.get_fixture(self.index_not_found_rs), }, ], "index": 2, "result": False, }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) response = es_client.delete_index(test["index"]) test["result"].should.equal(response) TestEsClient.shutdown_server(test["test_calls"])
def test_create_index(self): """Test creating index""" tests = [ { "test_calls": [ { "method": httpretty.PUT, "uri": "/idx0", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.index_created_rs), }, ], "index": "idx0", "acknowledged": True, }, { "test_calls": [ { "method": httpretty.PUT, "uri": "/idx1", "status": HTTPStatus.BAD_REQUEST, "content_type": "application/json", "rs": utils.get_fixture(self.index_already_exists_rs), }, ], "index": "idx1", "acknowledged": False, }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) response = es_client.create_index(test["index"]) response.acknowledged.should.equal(test["acknowledged"]) TestEsClient.shutdown_server(test["test_calls"])
def test_normalize_results(self): tests = [ { "elastic_results": [], "config": TestBoostingFeaturizer.get_default_config(), "result": [], }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": TestBoostingFeaturizer.get_default_config(), "result": [[{"_score": 158.08437, "normalized_score": 1.0, }]], }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))], "config": TestBoostingFeaturizer.get_default_config(), "result": [[{"_score": 158.08437, "normalized_score": 1.0, }, {"_score": 77.53298, "normalized_score": 0.4904, }, ]], }, ] weight_log_sim = weighted_similarity_calculator.\ WeightedSimilarityCalculator(folder=self.weights_folder) for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): _boosting_featurizer = BoostingFeaturizer( test["elastic_results"], test["config"], [], weighted_log_similarity_calculator=weight_log_sim) _boosting_featurizer.all_results.should.have.length_of(len(test["result"])) for i in range(len(test["result"])): for j in range(len(test["result"][i])): for field in test["result"][i][j]: elastic_res = _boosting_featurizer.all_results[i][1][j] elastic_res[field].should.equal(test["result"][i][j][field], epsilon=self.epsilon)
def test_exists_index(self): """Test existance of a index""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/idx0", "status": HTTPStatus.OK, }, ], "exists": True, "index": "idx0", }, { "test_calls": [ { "method": httpretty.GET, "uri": "/idx1", "status": HTTPStatus.NOT_FOUND, }, ], "exists": False, "index": "idx1", }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) response = es_client.index_exists(test["index"]) response.should.equal(test["exists"]) TestEsClient.shutdown_server(test["test_calls"])
def test_find_clusters(self): """Test finding clusters""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**(utils.get_fixture( self.launch_wo_test_items, to_json=True))[0]), for_update=False, numberOfLogLines=-1), "expected_result": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**(utils.get_fixture( self.launch_w_test_items_wo_logs, to_json=True))[0]), for_update=False, numberOfLogLines=-1), "expected_result": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, ], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**(utils.get_fixture( self.launch_w_test_items_w_empty_logs, to_json=True)[0] )), for_update=False, numberOfLogLines=-1), "expected_result": [] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=False, numberOfLogLines=-1), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update_all_the_same), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=False, numberOfLogLines=2), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="1") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_first_group), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_second_group), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=True, numberOfLogLines=-1), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_first_group), "rs": utils.get_fixture(self.one_hit_search_rs_clustering), }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_second_group), "rs": utils.get_fixture(self.one_hit_search_rs_clustering), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.cluster_update_es_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=True, numberOfLogLines=-1), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=111, testItemId=12, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="") ] }, { "test_calls": [{ "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/2/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_first_group_2lines), "rs": utils.get_fixture(self.one_hit_search_rs_clustering), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.cluster_update_all_the_same_es_update), "rs": utils.get_fixture(self.index_logs_rs), }], "launch_info": launch_objects.LaunchInfoForClustering( launch=launch_objects.Launch(**utils.get_fixture( self.launch_w_items_clustering, to_json=True)), for_update=True, numberOfLogLines=2), "expected_result": [ launch_objects.ClusterResult(logId=4, testItemId=2, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=5, testItemId=5, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=9, testItemId=6, project=2, launchId=1, clusterId="1"), launch_objects.ClusterResult(logId=111, testItemId=12, project=2, launchId=1, clusterId="1") ] }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) config = self.get_default_search_config() _cluster_service = ClusterService(app_config=self.app_config, search_cfg=config) response = _cluster_service.find_clusters(test["launch_info"]) response.should.have.length_of(len(test["expected_result"])) cluster_ids_dict = {} for i in range(len(response)): test["expected_result"][i].logId.should.equal( response[i].logId) if test["expected_result"][i].clusterId == "": test["expected_result"][i].clusterId.should.equal( response[i].clusterId) elif test["expected_result"][ i].clusterId not in cluster_ids_dict: cluster_ids_dict[test["expected_result"] [i].clusterId] = response[i].clusterId elif test["expected_result"][ i].clusterId in cluster_ids_dict: expected_cluster_id = cluster_ids_dict[ test["expected_result"][i].clusterId] expected_cluster_id.should.equal(response[i].clusterId) for cluster_id in cluster_ids_dict: test["test_calls"][-1]["rq"] = test["test_calls"][-1][ "rq"].replace( "\"cluster_id\":\"%s\"" % cluster_id, "\"cluster_id\":\"%s\"" % cluster_ids_dict[cluster_id]) TestClusterService.shutdown_server(test["test_calls"])
def test_filter_by_min_should_match_any(self): tests = [ { "elastic_results": [], "config": TestBoostingFeaturizer.get_default_config(filter_fields=[], filter_fields_any=[]), "result": [], }, { "elastic_results": [], "config": TestBoostingFeaturizer.get_default_config( filter_fields=[], filter_fields_any=["detected_message"]), "result": [], }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": TestBoostingFeaturizer.get_default_config( filter_fields=[], filter_fields_any=["detected_message", "detected_message_without_params_extended"]), "result": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)), (utils.get_fixture(self.log_message, to_json=True), utils.get_fixture( self.one_hit_search_rs_explained_wo_params, to_json=True))], "config": TestBoostingFeaturizer.get_default_config( filter_fields=[], filter_fields_any=["detected_message", "detected_message_without_params_extended"]), "result": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)), (utils.get_fixture(self.log_message, to_json=True), utils.get_fixture( self.one_hit_search_rs_explained_wo_params, to_json=True))] }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)), (utils.get_fixture(self.log_message, to_json=True), utils.get_fixture( self.one_hit_search_rs_explained_wo_params, to_json=True))], "config": TestBoostingFeaturizer.get_default_config( filter_fields=[], filter_fields_any=["detected_message"]), "result": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))] }, { "elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True), utils.get_fixture(self.one_hit_search_rs_small_logs, to_json=True))], "config": TestBoostingFeaturizer.get_default_config( filter_fields=[], filter_fields_any=["detected_message", "detected_message_without_params_extended"]), "result": [] }, { "elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True), utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))], "config": TestBoostingFeaturizer.get_default_config( filter_fields=[], filter_fields_any=["detected_message", "detected_message_without_params_extended"]), "result": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True), utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))] }, ] weight_log_sim = weighted_similarity_calculator.\ WeightedSimilarityCalculator(folder=self.weights_folder) for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): _boosting_featurizer = SuggestBoostingFeaturizer( test["elastic_results"], test["config"], [], weighted_log_similarity_calculator=weight_log_sim) all_results = test["elastic_results"] all_results = _boosting_featurizer.filter_by_min_should_match_any( all_results, fields=test["config"]["filter_min_should_match_any"]) all_results.should.have.length_of(len(test["result"])) for idx, (log, hits) in enumerate(all_results): log["_id"].should.equal(test["result"][idx][0]["_id"]) for i, hit in enumerate(hits["hits"]["hits"]): hit["_id"].should.equal(test["result"][idx][1]["hits"]["hits"][i]["_id"])
def test_find_most_relevant_by_type(self): tests = [ { "elastic_results": [], "config": TestBoostingFeaturizer.get_default_config(), "result": {}, }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.one_hit_search_rs_explained))], "config": TestBoostingFeaturizer.get_default_config(), "result": { "AB001": { "mrHit": { "_score": 158.08437, "_id": "1" }, "compared_log": self.get_fixture(self.log_message), "score": 1.0, }, } }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.two_hits_search_rs_explained))], "config": TestBoostingFeaturizer.get_default_config(), "result": { "AB001": { "mrHit": { "_score": 158.08437, "_id": "1" }, "compared_log": self.get_fixture(self.log_message), "score": 0.6709, }, "PB001": { "mrHit": { "_score": 77.53298, "_id": "2" }, "compared_log": self.get_fixture(self.log_message), "score": 0.3291, }, } }, { "elastic_results": [(self.get_fixture(self.log_message), self.get_fixture(self.two_hits_search_rs_explained)), (self.get_fixture(self.log_message), self.get_fixture(self.one_hit_search_rs_explained))], "config": TestBoostingFeaturizer.get_default_config(), "result": { "AB001": { "mrHit": { "_score": 158.08437, "_id": "1" }, "compared_log": self.get_fixture(self.log_message), "score": 0.8031, }, "PB001": { "mrHit": { "_score": 77.53298, "_id": "2" }, "compared_log": self.get_fixture(self.log_message), "score": 0.1969, }, } }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): _boosting_featurizer = BoostingFeaturizer( test["elastic_results"], test["config"], []) scores_by_issue_type = _boosting_featurizer.find_most_relevant_by_type( ) scores_by_issue_type.should.have.length_of(len(test["result"])) for issue_type in test["result"]: scores_by_issue_type.keys().should.contain(issue_type) elastic_res = scores_by_issue_type[issue_type] for field in test["result"][issue_type]: if type(test["result"][issue_type][field]) != dict: elastic_res[field].should.equal( test["result"][issue_type][field], epsilon=self.epsilon) else: for field_dict in test["result"][issue_type][ field]: result_field_dict = test["result"][issue_type][ field][field_dict] elastic_res[field][field_dict].should.equal( result_field_dict, epsilon=self.epsilon)
def test_full_data_check(self): print("Boost model folder : ", self.boost_model_folder) print("Weights model folder : ", self.weights_folder) print("Global defect type model folder : ", self.global_defect_type_model_folder) decision_maker = BoostingDecisionMaker(folder=self.boost_model_folder) boost_model_results = utils.get_fixture(self.boost_model_results, to_json=True) tests = [] for log_lines, filter_fields, _decision_maker in [ (-1, ["detected_message", "stacktrace"], decision_maker), (2, ["message"], decision_maker) ]: tests.extend([ { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": self.get_default_config(number_of_log_lines=log_lines, filter_fields=filter_fields), "decision_maker": _decision_maker }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))], "config": self.get_default_config(number_of_log_lines=log_lines, filter_fields=filter_fields), "decision_maker": _decision_maker }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True)), (utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": self.get_default_config(number_of_log_lines=log_lines, filter_fields=filter_fields), "decision_maker": _decision_maker }, { "elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True), utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))], "config": self.get_default_config(number_of_log_lines=log_lines, filter_fields=filter_fields), "decision_maker": _decision_maker }, ]) for idx, test in enumerate(tests): feature_ids = test["decision_maker"].get_feature_ids() weight_log_sim = None if self.weights_folder.strip(): weight_log_sim = weighted_similarity_calculator.\ WeightedSimilarityCalculator(folder=self.weights_folder) _boosting_featurizer = BoostingFeaturizer( test["elastic_results"], test["config"], feature_ids, weighted_log_similarity_calculator=weight_log_sim) if self.global_defect_type_model_folder.strip(): _boosting_featurizer.set_defect_type_model( defect_type_model.DefectTypeModel( folder=self.global_defect_type_model_folder)) with sure.ensure('Error in the test case number: {0}', idx): gathered_data, issue_type_names = _boosting_featurizer.gather_features_info( ) gathered_data.should.equal(boost_model_results[str(idx)][0], epsilon=self.epsilon) predict_label, predict_probability = test[ "decision_maker"].predict(gathered_data) predict_label.tolist().should.equal( boost_model_results[str(idx)][1], epsilon=self.epsilon) predict_probability.tolist().should.equal( boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_suggest_items(self): """Test suggesting test items""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "test_item_info": launch_objects.TestItemInfo(testItemId=1, uniqueId="341", testCaseHash=123, launchId=1, launchName="Launch", project=1, logs=[]), "expected_result": [], "boost_predict": ([], []) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, ], "test_item_info": launch_objects.TestItemInfo(testItemId=1, uniqueId="341", testCaseHash=123, launchId=1, launchName="Launch", project=2, logs=[ launch_objects.Log( logId=1, message="error found", logLevel=40000) ]), "expected_result": [], "boost_predict": ([], []) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "test_item_info": launch_objects.TestItemInfo(testItemId=1, uniqueId="341", testCaseHash=123, launchId=1, launchName="Launch", project=1, logs=[ launch_objects.Log( logId=1, message=" ", logLevel=40000) ]), "expected_result": [], "boost_predict": ([], []) }, { "test_calls": [{ "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.no_hits_search_rs), }], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [], "boost_predict": ([], []) }, { "test_calls": [{ "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.no_hits_search_rs), }], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [], "boost_predict": ([], []) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.no_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.one_hit_search_rs), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=80.0, esScore=10.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1], [[0.2, 0.8]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.one_hit_search_rs), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=70.0, esScore=10.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1], [[0.3, 0.7]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.two_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.two_hits_search_rs), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=70.0, esScore=15.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1, 0], [[0.3, 0.7], [0.9, 0.1]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.two_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.no_hits_search_rs), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=70.0, esScore=15.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80), launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='PB001', relevantItem=2, relevantLogId=2, matchScore=45.0, esScore=10.0, esPosition=1, modelFeatureNames='0', modelFeatureValues='0.67', modelInfo='', resultPosition=1, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1, 0], [[0.3, 0.7], [0.55, 0.45]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.two_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture(self.three_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.no_hits_search_rs), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='PB001', relevantItem=3, relevantLogId=3, matchScore=80.0, esScore=10.0, esPosition=2, modelFeatureNames='0', modelFeatureValues='0.67', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80), launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=70.0, esScore=15.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=1, usedLogLines=-1, minShouldMatch=80), launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='PB001', relevantItem=2, relevantLogId=2, matchScore=45.0, esScore=10.0, esPosition=1, modelFeatureNames='0', modelFeatureValues='0.67', modelInfo='', resultPosition=2, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1, 0, 1], [[0.3, 0.7], [0.55, 0.45], [0.2, 0.8]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_first), "rs": utils.get_fixture(self.two_hits_search_rs), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_second), "rs": utils.get_fixture( self.three_hits_search_rs_with_duplicate), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_third), "rs": utils.get_fixture(self.no_hits_search_rs), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=3, relevantLogId=3, matchScore=70.0, esScore=15.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80), launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=70.0, esScore=15.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=1, usedLogLines=-1, minShouldMatch=80), launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=178, issueType='PB001', relevantItem=2, relevantLogId=2, matchScore=70.0, esScore=10.0, esPosition=1, modelFeatureNames='0', modelFeatureValues='0.67', modelInfo='', resultPosition=2, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1, 1, 1], [[0.3, 0.7], [0.3, 0.7], [0.3, 0.7]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_merged_first), "rs": utils.get_fixture(self.one_hit_search_rs_merged), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_merged_second), "rs": utils.get_fixture(self.one_hit_search_rs_merged), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_merged_third), "rs": utils.get_fixture(self.one_hit_search_rs_merged), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_merged_logs, to_json=True)), "expected_result": [ launch_objects.SuggestAnalysisResult( testItem=123, testItemLogId=-1, issueType='AB001', relevantItem=1, relevantLogId=1, matchScore=90.0, esScore=10.0, esPosition=0, modelFeatureNames='0', modelFeatureValues='1.0', modelInfo='', resultPosition=0, usedLogLines=-1, minShouldMatch=80) ], "boost_predict": ([1], [[0.1, 0.9]]) }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_merged_first), "rs": utils.get_fixture(self.one_hit_search_rs_merged_wrong), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_merged_second), "rs": utils.get_fixture(self.one_hit_search_rs_merged_wrong), }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_rq_merged_third), "rs": utils.get_fixture(self.one_hit_search_rs_merged_wrong), }, ], "test_item_info": launch_objects.TestItemInfo(**utils.get_fixture( self.suggest_test_item_info_w_merged_logs, to_json=True)), "expected_result": [], "boost_predict": ([], []) }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) config = self.get_default_search_config() suggest_service = SuggestService(app_config=self.app_config, search_cfg=config) _boosting_decision_maker = BoostingDecisionMaker() _boosting_decision_maker.get_feature_ids = MagicMock( return_value=[0]) _boosting_decision_maker.predict = MagicMock( return_value=test["boost_predict"]) suggest_service.suggest_decision_maker = _boosting_decision_maker response = suggest_service.suggest_items( test["test_item_info"]) response.should.have.length_of(len(test["expected_result"])) for real_resp, expected_resp in zip(response, test["expected_result"]): real_resp.should.equal(expected_resp) TestSuggestService.shutdown_server(test["test_calls"])
def test_full_data_check_suggests(self): print("Boost model folder suggests: ", self.suggest_boost_model_folder) print("Weights model folder suggests: ", self.weights_folder) decision_maker = BoostingDecisionMaker( folder=self.suggest_boost_model_folder) boost_model_results = utils.get_fixture( self.suggest_boost_model_results, to_json=True) tests = [] all_configs = [(-1, [ "detected_message_extended", "detected_message_without_params_extended", "detected_message_without_params_and_brackets" ], decision_maker), (2, [ "message_extended", "message_without_params_extended", "message_without_params_and_brackets" ], decision_maker)] for log_lines, filter_fields_any, _decision_maker in all_configs: tests.extend([ { "elastic_results": [(utils.get_fixture(self.log_message_suggest, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": TestBoostingModel.get_default_config( number_of_log_lines=log_lines, filter_fields=[], filter_fields_any=filter_fields_any, min_should_match=0.4), "decision_maker": _decision_maker }, { "elastic_results": [(utils.get_fixture(self.log_message_suggest, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))], "config": TestBoostingModel.get_default_config( number_of_log_lines=log_lines, filter_fields=[], filter_fields_any=filter_fields_any, min_should_match=0.4), "decision_maker": _decision_maker }, { "elastic_results": [(utils.get_fixture(self.log_message_suggest, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True)), (utils.get_fixture(self.log_message_suggest, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": TestBoostingModel.get_default_config( number_of_log_lines=log_lines, filter_fields=[], filter_fields_any=filter_fields_any, min_should_match=0.4), "decision_maker": _decision_maker }, { "elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True), utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))], "config": TestBoostingModel.get_default_config( number_of_log_lines=log_lines, filter_fields=[], filter_fields_any=filter_fields_any, min_should_match=0.4), "decision_maker": _decision_maker }, ]) for idx, test in enumerate(tests): feature_ids = test["decision_maker"].get_feature_ids() weight_log_sim = None if self.weights_folder.strip(): weight_log_sim = weighted_similarity_calculator.\ WeightedSimilarityCalculator(folder=self.weights_folder) _boosting_featurizer = SuggestBoostingFeaturizer( test["elastic_results"], test["config"], feature_ids, weighted_log_similarity_calculator=weight_log_sim) with sure.ensure('Error in the test case number: {0}', idx): gathered_data, test_item_ids = _boosting_featurizer.gather_features_info( ) predict_label, predict_probability = test[ "decision_maker"].predict(gathered_data) gathered_data.should.equal(boost_model_results[str(idx)][0], epsilon=self.epsilon) predict_label.tolist().should.equal( boost_model_results[str(idx)][1], epsilon=self.epsilon) predict_probability.tolist().should.equal( boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_find_most_relevant_by_type_for_suggests(self): tests = [ { "elastic_results": [], "config": TestBoostingFeaturizer.get_default_config(), "result": {}, }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))], "config": TestBoostingFeaturizer.get_default_config(), "result": {1: {"mrHit": {"_score": 158.08437, "_id": "1"}, "compared_log": utils.get_fixture(self.log_message, to_json=True), "score": 1.0, }, } }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))], "config": TestBoostingFeaturizer.get_default_config(), "result": {1: {"mrHit": {"_score": 158.08437, "_id": "1"}, "compared_log": utils.get_fixture(self.log_message, to_json=True), "score": 1.0, }, 2: {"mrHit": {"_score": 77.53298, "_id": "2"}, "compared_log": utils.get_fixture(self.log_message, to_json=True), "score": 0.4905, }, } }, { "elastic_results": [(utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.two_hits_search_rs_explained, to_json=True)), (utils.get_fixture(self.log_message, to_json=True), utils.get_fixture(self.three_hits_search_rs_explained, to_json=True))], "config": TestBoostingFeaturizer.get_default_config(), "result": {1: {"mrHit": {"_score": 158.08437, "_id": "1"}, "compared_log": utils.get_fixture(self.log_message, to_json=True), "score": 0.9392, }, 2: {"mrHit": {"_score": 168.31, "_id": "2"}, "compared_log": utils.get_fixture(self.log_message, to_json=True), "score": 1.0, } } }, ] weight_log_sim = weighted_similarity_calculator.\ WeightedSimilarityCalculator(folder=self.weights_folder) for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): _boosting_featurizer = SuggestBoostingFeaturizer( test["elastic_results"], test["config"], [], weighted_log_similarity_calculator=weight_log_sim) scores_by_issue_type = _boosting_featurizer.find_most_relevant_by_type() scores_by_issue_type.should.have.length_of(len(test["result"])) for issue_type in test["result"]: scores_by_issue_type.keys().should.contain(issue_type) elastic_res = scores_by_issue_type[issue_type] for field in test["result"][issue_type]: if type(test["result"][issue_type][field]) != dict: elastic_res[field].should.equal(test["result"][issue_type][field], epsilon=self.epsilon) else: for field_dict in test["result"][issue_type][field]: result_field_dict = test["result"][issue_type][field][field_dict] elastic_res[field][field_dict].should.equal(result_field_dict, epsilon=self.epsilon)
def __test_something(): # same calculated value name = 'Hodor' with sure.ensure('the return value actually looks like: {0}', name): sure.expect(name).should.contain('whatever')
def validate_subnet_details_after_creating_eni(client, vpc, cidr, expected_ip_address_count): subnet = client.create_subnet(VpcId=vpc.id, CidrBlock=cidr, AvailabilityZone="us-west-1b")["Subnet"] # Create a random number of Elastic Network Interfaces nr_of_eni_to_create = random.randint(0, 5) ip_addresses_assigned = 0 enis_created = [] for i in range(0, nr_of_eni_to_create): # Create a random number of IP addresses per ENI nr_of_ip_addresses = random.randint(1, 5) if nr_of_ip_addresses == 1: # Pick the first available IP address (First 4 are reserved by AWS) private_address = "10.0.0." + str(ip_addresses_assigned + 4) eni = client.create_network_interface( SubnetId=subnet["SubnetId"], PrivateIpAddress=private_address)["NetworkInterface"] enis_created.append(eni) ip_addresses_assigned = ip_addresses_assigned + 1 else: # Assign a list of IP addresses private_addresses = [ "10.0.0." + str(4 + ip_addresses_assigned + i) for i in range(0, nr_of_ip_addresses) ] eni = client.create_network_interface( SubnetId=subnet["SubnetId"], PrivateIpAddresses=[{ "PrivateIpAddress": address } for address in private_addresses], )["NetworkInterface"] enis_created.append(eni) ip_addresses_assigned = ip_addresses_assigned + nr_of_ip_addresses + 1 # # Verify that the nr of available IP addresses takes these ENIs into account updated_subnet = client.describe_subnets( SubnetIds=[subnet["SubnetId"]])["Subnets"][0] private_addresses = [ eni["PrivateIpAddress"] for eni in enis_created if eni["PrivateIpAddress"] ] for eni in enis_created: private_addresses.extend([ address["PrivateIpAddress"] for address in eni["PrivateIpAddresses"] ]) error_msg = ( "Nr of IP addresses for Subnet with CIDR {0} is incorrect. Expected: {1}, Actual: {2}. " "Addresses: {3}") with sure.ensure( error_msg, cidr, str(expected_ip_address_count), updated_subnet["AvailableIpAddressCount"], str(private_addresses), ): updated_subnet["AvailableIpAddressCount"].should.equal( expected_ip_address_count - ip_addresses_assigned) # Clean up, as we have to create a few more subnets that shouldn't interfere with each other for eni in enis_created: client.delete_network_interface( NetworkInterfaceId=eni["NetworkInterfaceId"]) client.delete_subnet(SubnetId=subnet["SubnetId"])
def test_index_logs(self): """Test indexing logs from launches""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, "content_type": "application/json", }, ], "index_rq": utils.get_fixture(self.launch_wo_test_items), "has_errors": False, "expected_count": 0, "expected_log_exceptions": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, "content_type": "application/json", }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_wo_logs), "has_errors": False, "expected_count": 0, "expected_log_exceptions": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, "content_type": "application/json", }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_w_empty_logs), "has_errors": False, "expected_count": 0, "expected_log_exceptions": [] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, { "method": httpretty.PUT, "uri": "/2", "status": HTTPStatus.OK, "rs": utils.get_fixture(self.index_created_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.index_logs_rq_big_messages), "rs": utils.get_fixture(self.index_logs_rs), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_merged_logs), "rs": utils.get_fixture( self.two_hits_search_with_big_messages_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.delete_logs_rs), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_not_merged_logs), "rs": utils.get_fixture( self.two_hits_search_with_big_messages_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.index_logs_rq_merged_logs), "rs": utils.get_fixture(self.index_logs_rs), }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "has_errors": False, "expected_count": 2, "expected_log_exceptions": [ launch_objects.LogExceptionResult( logId=1, foundExceptions=['java.lang.NoClassDefFoundError']), launch_objects.LogExceptionResult( logId=2, foundExceptions=['java.lang.NoClassDefFoundError']) ] }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, { "method": httpretty.PUT, "uri": "/2", "status": HTTPStatus.OK, "rs": utils.get_fixture(self.index_created_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.index_logs_rq_different_log_level), "rs": utils.get_fixture( self.index_logs_rs_different_log_level), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_merged_logs), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.delete_logs_rs), }, { "method": httpretty.GET, "uri": "/2/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_not_merged_logs), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.index_logs_rq_different_log_level_merged), "rs": utils.get_fixture( self.index_logs_rs_different_log_level), }, ], "index_rq": utils.get_fixture( self.launch_w_test_items_w_logs_different_log_level), "has_errors": False, "expected_count": 1, "expected_log_exceptions": [ launch_objects.LogExceptionResult(logId=1, foundExceptions=[]) ] }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) es_client.es_client.scroll = MagicMock(return_value=json.loads( utils.get_fixture(self.no_hits_search_rs))) launches = [ launch_objects.Launch(**launch) for launch in json.loads(test["index_rq"]) ] response = es_client.index_logs(launches) test["has_errors"].should.equal(response.errors) test["expected_count"].should.equal(response.took) test["expected_log_exceptions"].should.equal( response.logResults) TestEsClient.shutdown_server(test["test_calls"])
def test_clean_index(self): """Test cleaning index logs""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture( self.search_not_merged_logs_for_delete), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.delete_logs_rs), }, { "method": httpretty.GET, "uri": "/1/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_merged_logs), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rs": utils.get_fixture(self.delete_logs_rs), }, { "method": httpretty.GET, "uri": "/1/_search?scroll=5m&size=1000", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_not_merged_logs), "rs": utils.get_fixture(self.one_hit_search_rs), }, { "method": httpretty.POST, "uri": "/_bulk?refresh=true", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.index_logs_rq), "rs": utils.get_fixture(self.index_logs_rs), }, ], "rq": launch_objects.CleanIndex(ids=[1], project=1), "expected_count": 1 }, { "test_calls": [ { "method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, ], "rq": launch_objects.CleanIndex(ids=[1], project=2), "expected_count": 0 }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) es_client = esclient.EsClient( app_config=self.app_config, search_cfg=self.get_default_search_config()) es_client.es_client.scroll = MagicMock(return_value=json.loads( utils.get_fixture(self.no_hits_search_rs))) response = es_client.delete_logs(test["rq"]) test["expected_count"].should.equal(response) TestEsClient.shutdown_server(test["test_calls"])
def __test_something(): # same calculated value with sure.ensure('neverused'): raise Exception('This is not an AssertionError')
def step_impl(context): do_build = os.path.join(context.fake_srcdir, 'do', 'build') with sure.ensure('do/build should exist'): os.path.exists(do_build).should.be.true with sure.ensure('do/build should be executable'): os.access(do_build, os.X_OK).should.be.true
def test_analyze_logs(self): """Test analyzing logs""" tests = [ { "test_calls": [{"method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "index_rq": utils.get_fixture(self.launch_wo_test_items), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "index_rq": utils.get_fixture( self.launch_w_test_items_wo_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, ], "index_rq": utils.get_fixture( self.launch_w_test_items_w_empty_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }, ], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.no_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.NOT_FOUND, }, ], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.one_hit_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1], [[0.2, 0.8]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.one_hit_search_rs, to_json=True), utils.get_fixture(self.two_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1, 1], [[0.2, 0.8], [0.3, 0.7]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "PB001", "boost_predict": ([0, 1], [[0.8, 0.2], [0.3, 0.7]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture(self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]]) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True)], "index_rq": utils.get_fixture( self.launch_w_test_items_w_logs_to_be_merged), "expected_count": 0, "expected_issue_type": "", "boost_predict": ([], []) }, { "test_calls": [{"method": httpretty.GET, "uri": "/2", "status": HTTPStatus.OK, }], "msearch_results": [ utils.get_fixture(self.no_hits_search_rs, to_json=True), utils.get_fixture(self.three_hits_search_rs_with_one_unique_id, to_json=True)], "index_rq": utils.get_fixture( self.launch_w_test_items_w_logs), "expected_count": 1, "expected_issue_type": "AB001", "boost_predict": ([1], [[0.2, 0.8]]) } ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) config = self.get_default_search_config() analyzer_service = AutoAnalyzerService(app_config=self.app_config, search_cfg=config) _boosting_decision_maker = BoostingDecisionMaker() _boosting_decision_maker.get_feature_ids = MagicMock(return_value=[0]) _boosting_decision_maker.predict = MagicMock(return_value=test["boost_predict"]) if "msearch_results" in test: analyzer_service.es_client.es_client.msearch = MagicMock( return_value={"responses": test["msearch_results"]}) analyzer_service.boosting_decision_maker = _boosting_decision_maker launches = [launch_objects.Launch(**launch) for launch in json.loads(test["index_rq"])] response = analyzer_service.analyze_logs(launches) response.should.have.length_of(test["expected_count"]) if test["expected_issue_type"] != "": test["expected_issue_type"].should.equal(response[0].issueType) if "expected_id" in test: test["expected_id"].should.equal(response[0].relevantItem) TestAutoAnalyzerService.shutdown_server(test["test_calls"])
def test_search_logs(self): """Test search logs""" tests = [ { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq), "rs": utils.get_fixture(self.no_hits_search_rs), }, ], "rq": launch_objects.SearchLogs(launchId=1, launchName="Launch 1", itemId=3, projectId=1, filteredLaunchIds=[1], logMessages=["error"], logLines=-1), "expected_count": 0 }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, ], "rq": launch_objects.SearchLogs(launchId=1, launchName="Launch 1", itemId=3, projectId=1, filteredLaunchIds=[1], logMessages=[""], logLines=-1), "expected_count": 0 }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq), "rs": utils.get_fixture(self.one_hit_search_rs_search_logs), }, ], "rq": launch_objects.SearchLogs(launchId=1, launchName="Launch 1", itemId=3, projectId=1, filteredLaunchIds=[1], logMessages=["error"], logLines=-1), "expected_count": 0 }, { "test_calls": [ { "method": httpretty.GET, "uri": "/1", "status": HTTPStatus.OK, }, { "method": httpretty.GET, "uri": "/1/_search", "status": HTTPStatus.OK, "content_type": "application/json", "rq": utils.get_fixture(self.search_logs_rq_not_found), "rs": utils.get_fixture(self.two_hits_search_rs_search_logs), }, ], "rq": launch_objects.SearchLogs(launchId=1, launchName="Launch 1", itemId=3, projectId=1, filteredLaunchIds=[1], logMessages=["error occured once"], logLines=-1), "expected_count": 1 }, ] for idx, test in enumerate(tests): with sure.ensure('Error in the test case number: {0}', idx): self._start_server(test["test_calls"]) search_service = SearchService( app_config=self.app_config, search_cfg=self.get_default_search_config()) response = search_service.search_logs(test["rq"]) response.should.have.length_of(test["expected_count"]) TestSearchService.shutdown_server(test["test_calls"])