def test_create_custom_lambda_resource():
#########
# Integration test using a Custom Resource
# Create a Lambda
# CF will call the Lambda
# The Lambda should call CF, to indicate success (using the cfnresponse-module)
# This HTTP request will include any outputs that are now stored against the stack
# TEST: verify that this output is persisted
##########
if not settings.TEST_SERVER_MODE:
raise SkipTest(
"Needs a standalone MotoServer, as cfnresponse needs to connect to something"
)
# Create cloudformation stack
stack_name = f"stack{str(uuid4())[0:6]}"
template_body = get_template(get_lambda_code())
cf = boto3.client("cloudformation", region_name="us-east-1")
cf.create_stack(
StackName=stack_name,
TemplateBody=json.dumps(template_body),
Capabilities=["CAPABILITY_IAM"],
)
# Verify CloudWatch contains the correct logs
log_group_name = get_log_group_name(cf, stack_name)
success, logs = wait_for_log_msg(expected_msg="Status code: 200",
log_group=log_group_name)
with sure.ensure(f"Logs should indicate success: \n{logs}"):
success.should.equal(True)
# Verify the correct Output was returned
outputs = get_outputs(cf, stack_name)
outputs.should.have.length_of(1)
outputs[0].should.have.key("OutputKey").equals("infokey")
outputs[0].should.have.key("OutputValue").equals("special value")
def video(self, allow_redirects=False, size_limit=None):
r = self.obj.should.be.downloadable(allow_redirects=allow_redirects)
if size_limit:
r.headers.should.have.key('Content-Length')
source_size = int(r.headers['Content-Length'])
size_limit = int(size_limit)
with ensure(
"source files size ({}) should be less limit size ({})".format(
humanize.naturalsize(source_size),
humanize.naturalsize(size_limit))):
source_size.should.be.lower_than(size_limit)
return r
def test_list_indices(self):
"""Test checking getting indices from elasticsearch"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/_cat/indices?format=json",
"status": HTTPStatus.OK,
"rs": "[]",
},
],
"expected_count":
0,
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/_cat/indices?format=json",
"status": HTTPStatus.OK,
"rs": utils.get_fixture(self.two_indices_rs),
},
],
"expected_count":
2,
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/_cat/indices?format=json",
"status": HTTPStatus.INTERNAL_SERVER_ERROR,
},
],
"expected_count":
0,
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.list_indices()
response.should.have.length_of(test["expected_count"])
TestEsClient.shutdown_server(test["test_calls"])
def test_full_data_check(self):
print("Boost model folder: ", self.boost_model_folder)
decision_maker = BoostingDecisionMaker(self.boost_model_folder)
boost_model_results = self.get_fixture(self.boost_model_results)
tests = [
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.one_hit_search_rs_explained))],
"config":
TestBoostingModel.get_default_config(),
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.two_hits_search_rs_explained))],
"config":
TestBoostingModel.get_default_config(),
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.two_hits_search_rs_explained)),
(self.get_fixture(self.log_message),
self.get_fixture(self.one_hit_search_rs_explained))],
"config":
TestBoostingModel.get_default_config(),
},
{
"elastic_results":
[(self.get_fixture(self.log_message_only_small_logs),
self.get_fixture(self.two_hits_search_rs_small_logs))],
"config":
TestBoostingModel.get_default_config(),
},
]
for idx, test in enumerate(tests):
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"], test["config"],
decision_maker.get_feature_ids())
with sure.ensure('Error in the test case number: {0}', idx):
gathered_data, issue_type_names = _boosting_featurizer.gather_features_info(
)
gathered_data.should.equal(boost_model_results[str(idx)][0],
epsilon=self.epsilon)
predict_label, predict_probability = decision_maker.predict(
gathered_data)
predict_label.tolist().should.equal(
boost_model_results[str(idx)][1], epsilon=self.epsilon)
predict_probability.tolist().should.equal(
boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_normalize_results(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [],
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.one_hit_search_rs_explained))],
"config":
TestBoostingFeaturizer.get_default_config(),
"result": [[{
"_score": 158.08437,
"normalized_score": 1.0,
}]],
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.two_hits_search_rs_explained))],
"config":
TestBoostingFeaturizer.get_default_config(),
"result": [[
{
"_score": 158.08437,
"normalized_score": 1.0,
},
{
"_score": 77.53298,
"normalized_score": 0.4904,
},
]],
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"], test["config"], [])
_boosting_featurizer.all_results.should.have.length_of(
len(test["result"]))
for i in range(len(test["result"])):
for j in range(len(test["result"][i])):
for field in test["result"][i][j]:
elastic_res = _boosting_featurizer.all_results[i][
1][j]
elastic_res[field].should.equal(
test["result"][i][j][field],
epsilon=self.epsilon)
def test_delete_index(self):
"""Test deleting an index"""
tests = [
{
"test_calls": [
{
"method": httpretty.DELETE,
"uri": "/1",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_deleted_rs),
},
],
"index":
1,
"result":
True,
},
{
"test_calls": [
{
"method": httpretty.DELETE,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_not_found_rs),
},
],
"index":
2,
"result":
False,
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.delete_index(test["index"])
test["result"].should.equal(response)
TestEsClient.shutdown_server(test["test_calls"])
def test_create_index(self):
"""Test creating index"""
tests = [
{
"test_calls": [
{
"method": httpretty.PUT,
"uri": "/idx0",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_created_rs),
},
],
"index":
"idx0",
"acknowledged":
True,
},
{
"test_calls": [
{
"method": httpretty.PUT,
"uri": "/idx1",
"status": HTTPStatus.BAD_REQUEST,
"content_type": "application/json",
"rs": utils.get_fixture(self.index_already_exists_rs),
},
],
"index":
"idx1",
"acknowledged":
False,
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.create_index(test["index"])
response.acknowledged.should.equal(test["acknowledged"])
TestEsClient.shutdown_server(test["test_calls"])
def test_normalize_results(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [[{"_score": 158.08437,
"normalized_score": 1.0, }]],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": [[{"_score": 158.08437,
"normalized_score": 1.0,
},
{"_score": 77.53298,
"normalized_score": 0.4904,
}, ]],
},
]
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"],
test["config"],
[],
weighted_log_similarity_calculator=weight_log_sim)
_boosting_featurizer.all_results.should.have.length_of(len(test["result"]))
for i in range(len(test["result"])):
for j in range(len(test["result"][i])):
for field in test["result"][i][j]:
elastic_res = _boosting_featurizer.all_results[i][1][j]
elastic_res[field].should.equal(test["result"][i][j][field],
epsilon=self.epsilon)
def test_exists_index(self):
"""Test existance of a index"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/idx0",
"status": HTTPStatus.OK,
},
],
"exists":
True,
"index":
"idx0",
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/idx1",
"status": HTTPStatus.NOT_FOUND,
},
],
"exists":
False,
"index":
"idx1",
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = es_client.index_exists(test["index"])
response.should.equal(test["exists"])
TestEsClient.shutdown_server(test["test_calls"])
def test_find_clusters(self):
"""Test finding clusters"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**(utils.get_fixture(
self.launch_wo_test_items, to_json=True))[0]),
for_update=False,
numberOfLogLines=-1),
"expected_result": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**(utils.get_fixture(
self.launch_w_test_items_wo_logs, to_json=True))[0]),
for_update=False,
numberOfLogLines=-1),
"expected_result": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
},
],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**(utils.get_fixture(
self.launch_w_test_items_w_empty_logs, to_json=True)[0]
)),
for_update=False,
numberOfLogLines=-1),
"expected_result": []
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.cluster_update),
"rs": utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=False,
numberOfLogLines=-1),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.cluster_update_all_the_same),
"rs":
utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=False,
numberOfLogLines=2),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="1")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_first_group),
"rs":
utils.get_fixture(self.no_hits_search_rs),
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_second_group),
"rs":
utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.cluster_update),
"rs": utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=True,
numberOfLogLines=-1),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_first_group),
"rs":
utils.get_fixture(self.one_hit_search_rs_clustering),
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_second_group),
"rs":
utils.get_fixture(self.one_hit_search_rs_clustering),
}, {
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.cluster_update_es_update),
"rs":
utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=True,
numberOfLogLines=-1),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=111,
testItemId=12,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="")
]
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, {
"method":
httpretty.GET,
"uri":
"/2/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_first_group_2lines),
"rs":
utils.get_fixture(self.one_hit_search_rs_clustering),
}, {
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.cluster_update_all_the_same_es_update),
"rs":
utils.get_fixture(self.index_logs_rs),
}],
"launch_info":
launch_objects.LaunchInfoForClustering(
launch=launch_objects.Launch(**utils.get_fixture(
self.launch_w_items_clustering, to_json=True)),
for_update=True,
numberOfLogLines=2),
"expected_result": [
launch_objects.ClusterResult(logId=4,
testItemId=2,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=5,
testItemId=5,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=9,
testItemId=6,
project=2,
launchId=1,
clusterId="1"),
launch_objects.ClusterResult(logId=111,
testItemId=12,
project=2,
launchId=1,
clusterId="1")
]
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
_cluster_service = ClusterService(app_config=self.app_config,
search_cfg=config)
response = _cluster_service.find_clusters(test["launch_info"])
response.should.have.length_of(len(test["expected_result"]))
cluster_ids_dict = {}
for i in range(len(response)):
test["expected_result"][i].logId.should.equal(
response[i].logId)
if test["expected_result"][i].clusterId == "":
test["expected_result"][i].clusterId.should.equal(
response[i].clusterId)
elif test["expected_result"][
i].clusterId not in cluster_ids_dict:
cluster_ids_dict[test["expected_result"]
[i].clusterId] = response[i].clusterId
elif test["expected_result"][
i].clusterId in cluster_ids_dict:
expected_cluster_id = cluster_ids_dict[
test["expected_result"][i].clusterId]
expected_cluster_id.should.equal(response[i].clusterId)
for cluster_id in cluster_ids_dict:
test["test_calls"][-1]["rq"] = test["test_calls"][-1][
"rq"].replace(
"\"cluster_id\":\"%s\"" % cluster_id,
"\"cluster_id\":\"%s\"" %
cluster_ids_dict[cluster_id])
TestClusterService.shutdown_server(test["test_calls"])
def test_filter_by_min_should_match_any(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(filter_fields=[],
filter_fields_any=[]),
"result": [],
},
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message"]),
"result": [],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(
self.one_hit_search_rs_explained_wo_params, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(
self.one_hit_search_rs_explained_wo_params, to_json=True))]
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(
self.one_hit_search_rs_explained_wo_params, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message"]),
"result": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))]
},
{
"elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True),
utils.get_fixture(self.one_hit_search_rs_small_logs, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": []
},
{
"elastic_results": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(
filter_fields=[],
filter_fields_any=["detected_message",
"detected_message_without_params_extended"]),
"result": [(utils.get_fixture(self.log_message_only_small_logs, to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs, to_json=True))]
},
]
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = SuggestBoostingFeaturizer(
test["elastic_results"],
test["config"],
[],
weighted_log_similarity_calculator=weight_log_sim)
all_results = test["elastic_results"]
all_results = _boosting_featurizer.filter_by_min_should_match_any(
all_results,
fields=test["config"]["filter_min_should_match_any"])
all_results.should.have.length_of(len(test["result"]))
for idx, (log, hits) in enumerate(all_results):
log["_id"].should.equal(test["result"][idx][0]["_id"])
for i, hit in enumerate(hits["hits"]["hits"]):
hit["_id"].should.equal(test["result"][idx][1]["hits"]["hits"][i]["_id"])
def test_find_most_relevant_by_type(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {},
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.one_hit_search_rs_explained))],
"config":
TestBoostingFeaturizer.get_default_config(),
"result": {
"AB001": {
"mrHit": {
"_score": 158.08437,
"_id": "1"
},
"compared_log": self.get_fixture(self.log_message),
"score": 1.0,
},
}
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.two_hits_search_rs_explained))],
"config":
TestBoostingFeaturizer.get_default_config(),
"result": {
"AB001": {
"mrHit": {
"_score": 158.08437,
"_id": "1"
},
"compared_log": self.get_fixture(self.log_message),
"score": 0.6709,
},
"PB001": {
"mrHit": {
"_score": 77.53298,
"_id": "2"
},
"compared_log": self.get_fixture(self.log_message),
"score": 0.3291,
},
}
},
{
"elastic_results":
[(self.get_fixture(self.log_message),
self.get_fixture(self.two_hits_search_rs_explained)),
(self.get_fixture(self.log_message),
self.get_fixture(self.one_hit_search_rs_explained))],
"config":
TestBoostingFeaturizer.get_default_config(),
"result": {
"AB001": {
"mrHit": {
"_score": 158.08437,
"_id": "1"
},
"compared_log": self.get_fixture(self.log_message),
"score": 0.8031,
},
"PB001": {
"mrHit": {
"_score": 77.53298,
"_id": "2"
},
"compared_log": self.get_fixture(self.log_message),
"score": 0.1969,
},
}
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"], test["config"], [])
scores_by_issue_type = _boosting_featurizer.find_most_relevant_by_type(
)
scores_by_issue_type.should.have.length_of(len(test["result"]))
for issue_type in test["result"]:
scores_by_issue_type.keys().should.contain(issue_type)
elastic_res = scores_by_issue_type[issue_type]
for field in test["result"][issue_type]:
if type(test["result"][issue_type][field]) != dict:
elastic_res[field].should.equal(
test["result"][issue_type][field],
epsilon=self.epsilon)
else:
for field_dict in test["result"][issue_type][
field]:
result_field_dict = test["result"][issue_type][
field][field_dict]
elastic_res[field][field_dict].should.equal(
result_field_dict, epsilon=self.epsilon)
def test_full_data_check(self):
print("Boost model folder : ", self.boost_model_folder)
print("Weights model folder : ", self.weights_folder)
print("Global defect type model folder : ",
self.global_defect_type_model_folder)
decision_maker = BoostingDecisionMaker(folder=self.boost_model_folder)
boost_model_results = utils.get_fixture(self.boost_model_results,
to_json=True)
tests = []
for log_lines, filter_fields, _decision_maker in [
(-1, ["detected_message", "stacktrace"], decision_maker),
(2, ["message"], decision_maker)
]:
tests.extend([
{
"elastic_results":
[(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_only_small_logs,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs,
to_json=True))],
"config":
self.get_default_config(number_of_log_lines=log_lines,
filter_fields=filter_fields),
"decision_maker":
_decision_maker
},
])
for idx, test in enumerate(tests):
feature_ids = test["decision_maker"].get_feature_ids()
weight_log_sim = None
if self.weights_folder.strip():
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
_boosting_featurizer = BoostingFeaturizer(
test["elastic_results"],
test["config"],
feature_ids,
weighted_log_similarity_calculator=weight_log_sim)
if self.global_defect_type_model_folder.strip():
_boosting_featurizer.set_defect_type_model(
defect_type_model.DefectTypeModel(
folder=self.global_defect_type_model_folder))
with sure.ensure('Error in the test case number: {0}', idx):
gathered_data, issue_type_names = _boosting_featurizer.gather_features_info(
)
gathered_data.should.equal(boost_model_results[str(idx)][0],
epsilon=self.epsilon)
predict_label, predict_probability = test[
"decision_maker"].predict(gathered_data)
predict_label.tolist().should.equal(
boost_model_results[str(idx)][1], epsilon=self.epsilon)
predict_probability.tolist().should.equal(
boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_suggest_items(self):
"""Test suggesting test items"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=1,
logs=[]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=2,
logs=[
launch_objects.Log(
logId=1,
message="error found",
logLevel=40000)
]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"test_item_info":
launch_objects.TestItemInfo(testItemId=1,
uniqueId="341",
testCaseHash=123,
launchId=1,
launchName="Launch",
project=1,
logs=[
launch_objects.Log(
logId=1,
message=" ",
logLevel=40000)
]),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
}],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.no_hits_search_rs),
}, {
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
}],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=80.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.2, 0.8]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.3, 0.7]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0], [[0.3, 0.7], [0.9, 0.1]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=45.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0], [[0.3, 0.7], [0.55, 0.45]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_second),
"rs": utils.get_fixture(self.three_hits_search_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=3,
relevantLogId=3,
matchScore=80.0,
esScore=10.0,
esPosition=2,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=45.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=2,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 0, 1], [[0.3, 0.7], [0.55, 0.45],
[0.2, 0.8]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_first),
"rs": utils.get_fixture(self.two_hits_search_rs),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_second),
"rs":
utils.get_fixture(
self.three_hits_search_rs_with_duplicate),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_third),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=3,
relevantLogId=3,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=70.0,
esScore=15.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=1,
usedLogLines=-1,
minShouldMatch=80),
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=178,
issueType='PB001',
relevantItem=2,
relevantLogId=2,
matchScore=70.0,
esScore=10.0,
esPosition=1,
modelFeatureNames='0',
modelFeatureValues='0.67',
modelInfo='',
resultPosition=2,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1, 1, 1], [[0.3, 0.7], [0.3, 0.7],
[0.3, 0.7]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_first),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_second),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_rq_merged_third),
"rs": utils.get_fixture(self.one_hit_search_rs_merged),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_merged_logs, to_json=True)),
"expected_result": [
launch_objects.SuggestAnalysisResult(
testItem=123,
testItemLogId=-1,
issueType='AB001',
relevantItem=1,
relevantLogId=1,
matchScore=90.0,
esScore=10.0,
esPosition=0,
modelFeatureNames='0',
modelFeatureValues='1.0',
modelInfo='',
resultPosition=0,
usedLogLines=-1,
minShouldMatch=80)
],
"boost_predict": ([1], [[0.1, 0.9]])
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_first),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_second),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_rq_merged_third),
"rs":
utils.get_fixture(self.one_hit_search_rs_merged_wrong),
},
],
"test_item_info":
launch_objects.TestItemInfo(**utils.get_fixture(
self.suggest_test_item_info_w_merged_logs, to_json=True)),
"expected_result": [],
"boost_predict": ([], [])
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
suggest_service = SuggestService(app_config=self.app_config,
search_cfg=config)
_boosting_decision_maker = BoostingDecisionMaker()
_boosting_decision_maker.get_feature_ids = MagicMock(
return_value=[0])
_boosting_decision_maker.predict = MagicMock(
return_value=test["boost_predict"])
suggest_service.suggest_decision_maker = _boosting_decision_maker
response = suggest_service.suggest_items(
test["test_item_info"])
response.should.have.length_of(len(test["expected_result"]))
for real_resp, expected_resp in zip(response,
test["expected_result"]):
real_resp.should.equal(expected_resp)
TestSuggestService.shutdown_server(test["test_calls"])
def test_full_data_check_suggests(self):
print("Boost model folder suggests: ", self.suggest_boost_model_folder)
print("Weights model folder suggests: ", self.weights_folder)
decision_maker = BoostingDecisionMaker(
folder=self.suggest_boost_model_folder)
boost_model_results = utils.get_fixture(
self.suggest_boost_model_results, to_json=True)
tests = []
all_configs = [(-1, [
"detected_message_extended",
"detected_message_without_params_extended",
"detected_message_without_params_and_brackets"
], decision_maker),
(2, [
"message_extended",
"message_without_params_extended",
"message_without_params_and_brackets"
], decision_maker)]
for log_lines, filter_fields_any, _decision_maker in all_configs:
tests.extend([
{
"elastic_results":
[(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained,
to_json=True)),
(utils.get_fixture(self.log_message_suggest,
to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
{
"elastic_results":
[(utils.get_fixture(self.log_message_only_small_logs,
to_json=True),
utils.get_fixture(self.two_hits_search_rs_small_logs,
to_json=True))],
"config":
TestBoostingModel.get_default_config(
number_of_log_lines=log_lines,
filter_fields=[],
filter_fields_any=filter_fields_any,
min_should_match=0.4),
"decision_maker":
_decision_maker
},
])
for idx, test in enumerate(tests):
feature_ids = test["decision_maker"].get_feature_ids()
weight_log_sim = None
if self.weights_folder.strip():
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
_boosting_featurizer = SuggestBoostingFeaturizer(
test["elastic_results"],
test["config"],
feature_ids,
weighted_log_similarity_calculator=weight_log_sim)
with sure.ensure('Error in the test case number: {0}', idx):
gathered_data, test_item_ids = _boosting_featurizer.gather_features_info(
)
predict_label, predict_probability = test[
"decision_maker"].predict(gathered_data)
gathered_data.should.equal(boost_model_results[str(idx)][0],
epsilon=self.epsilon)
predict_label.tolist().should.equal(
boost_model_results[str(idx)][1], epsilon=self.epsilon)
predict_probability.tolist().should.equal(
boost_model_results[str(idx)][2], epsilon=self.epsilon)
def test_find_most_relevant_by_type_for_suggests(self):
tests = [
{
"elastic_results": [],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {},
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.one_hit_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {1: {"mrHit": {"_score": 158.08437,
"_id": "1"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 1.0, },
}
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {1: {"mrHit": {"_score": 158.08437,
"_id": "1"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 1.0, },
2: {"mrHit": {"_score": 77.53298,
"_id": "2"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 0.4905, },
}
},
{
"elastic_results": [(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.two_hits_search_rs_explained, to_json=True)),
(utils.get_fixture(self.log_message, to_json=True),
utils.get_fixture(self.three_hits_search_rs_explained, to_json=True))],
"config": TestBoostingFeaturizer.get_default_config(),
"result": {1: {"mrHit": {"_score": 158.08437,
"_id": "1"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 0.9392, },
2: {"mrHit": {"_score": 168.31,
"_id": "2"},
"compared_log": utils.get_fixture(self.log_message, to_json=True),
"score": 1.0, }
}
},
]
weight_log_sim = weighted_similarity_calculator.\
WeightedSimilarityCalculator(folder=self.weights_folder)
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
_boosting_featurizer = SuggestBoostingFeaturizer(
test["elastic_results"],
test["config"],
[],
weighted_log_similarity_calculator=weight_log_sim)
scores_by_issue_type = _boosting_featurizer.find_most_relevant_by_type()
scores_by_issue_type.should.have.length_of(len(test["result"]))
for issue_type in test["result"]:
scores_by_issue_type.keys().should.contain(issue_type)
elastic_res = scores_by_issue_type[issue_type]
for field in test["result"][issue_type]:
if type(test["result"][issue_type][field]) != dict:
elastic_res[field].should.equal(test["result"][issue_type][field],
epsilon=self.epsilon)
else:
for field_dict in test["result"][issue_type][field]:
result_field_dict = test["result"][issue_type][field][field_dict]
elastic_res[field][field_dict].should.equal(result_field_dict,
epsilon=self.epsilon)
def __test_something():
# same calculated value
name = 'Hodor'
with sure.ensure('the return value actually looks like: {0}', name):
sure.expect(name).should.contain('whatever')
def validate_subnet_details_after_creating_eni(client, vpc, cidr,
expected_ip_address_count):
subnet = client.create_subnet(VpcId=vpc.id,
CidrBlock=cidr,
AvailabilityZone="us-west-1b")["Subnet"]
# Create a random number of Elastic Network Interfaces
nr_of_eni_to_create = random.randint(0, 5)
ip_addresses_assigned = 0
enis_created = []
for i in range(0, nr_of_eni_to_create):
# Create a random number of IP addresses per ENI
nr_of_ip_addresses = random.randint(1, 5)
if nr_of_ip_addresses == 1:
# Pick the first available IP address (First 4 are reserved by AWS)
private_address = "10.0.0." + str(ip_addresses_assigned + 4)
eni = client.create_network_interface(
SubnetId=subnet["SubnetId"],
PrivateIpAddress=private_address)["NetworkInterface"]
enis_created.append(eni)
ip_addresses_assigned = ip_addresses_assigned + 1
else:
# Assign a list of IP addresses
private_addresses = [
"10.0.0." + str(4 + ip_addresses_assigned + i)
for i in range(0, nr_of_ip_addresses)
]
eni = client.create_network_interface(
SubnetId=subnet["SubnetId"],
PrivateIpAddresses=[{
"PrivateIpAddress": address
} for address in private_addresses],
)["NetworkInterface"]
enis_created.append(eni)
ip_addresses_assigned = ip_addresses_assigned + nr_of_ip_addresses + 1 #
# Verify that the nr of available IP addresses takes these ENIs into account
updated_subnet = client.describe_subnets(
SubnetIds=[subnet["SubnetId"]])["Subnets"][0]
private_addresses = [
eni["PrivateIpAddress"] for eni in enis_created
if eni["PrivateIpAddress"]
]
for eni in enis_created:
private_addresses.extend([
address["PrivateIpAddress"]
for address in eni["PrivateIpAddresses"]
])
error_msg = (
"Nr of IP addresses for Subnet with CIDR {0} is incorrect. Expected: {1}, Actual: {2}. "
"Addresses: {3}")
with sure.ensure(
error_msg,
cidr,
str(expected_ip_address_count),
updated_subnet["AvailableIpAddressCount"],
str(private_addresses),
):
updated_subnet["AvailableIpAddressCount"].should.equal(
expected_ip_address_count - ip_addresses_assigned)
# Clean up, as we have to create a few more subnets that shouldn't interfere with each other
for eni in enis_created:
client.delete_network_interface(
NetworkInterfaceId=eni["NetworkInterfaceId"])
client.delete_subnet(SubnetId=subnet["SubnetId"])
def test_index_logs(self):
"""Test indexing logs from launches"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
"content_type": "application/json",
},
],
"index_rq":
utils.get_fixture(self.launch_wo_test_items),
"has_errors":
False,
"expected_count":
0,
"expected_log_exceptions": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
"content_type": "application/json",
},
],
"index_rq":
utils.get_fixture(self.launch_w_test_items_wo_logs),
"has_errors":
False,
"expected_count":
0,
"expected_log_exceptions": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
"content_type": "application/json",
},
],
"index_rq":
utils.get_fixture(self.launch_w_test_items_w_empty_logs),
"has_errors":
False,
"expected_count":
0,
"expected_log_exceptions": []
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
{
"method": httpretty.PUT,
"uri": "/2",
"status": HTTPStatus.OK,
"rs": utils.get_fixture(self.index_created_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq":
utils.get_fixture(self.index_logs_rq_big_messages),
"rs": utils.get_fixture(self.index_logs_rs),
},
{
"method":
httpretty.GET,
"uri":
"/2/_search?scroll=5m&size=1000",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_merged_logs),
"rs":
utils.get_fixture(
self.two_hits_search_with_big_messages_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method":
httpretty.GET,
"uri":
"/2/_search?scroll=5m&size=1000",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_not_merged_logs),
"rs":
utils.get_fixture(
self.two_hits_search_with_big_messages_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq":
utils.get_fixture(self.index_logs_rq_merged_logs),
"rs": utils.get_fixture(self.index_logs_rs),
},
],
"index_rq":
utils.get_fixture(self.launch_w_test_items_w_logs),
"has_errors":
False,
"expected_count":
2,
"expected_log_exceptions": [
launch_objects.LogExceptionResult(
logId=1,
foundExceptions=['java.lang.NoClassDefFoundError']),
launch_objects.LogExceptionResult(
logId=2,
foundExceptions=['java.lang.NoClassDefFoundError'])
]
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
{
"method": httpretty.PUT,
"uri": "/2",
"status": HTTPStatus.OK,
"rs": utils.get_fixture(self.index_created_rs),
},
{
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.index_logs_rq_different_log_level),
"rs":
utils.get_fixture(
self.index_logs_rs_different_log_level),
},
{
"method": httpretty.GET,
"uri": "/2/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method": httpretty.GET,
"uri": "/2/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_not_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method":
httpretty.POST,
"uri":
"/_bulk?refresh=true",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.index_logs_rq_different_log_level_merged),
"rs":
utils.get_fixture(
self.index_logs_rs_different_log_level),
},
],
"index_rq":
utils.get_fixture(
self.launch_w_test_items_w_logs_different_log_level),
"has_errors":
False,
"expected_count":
1,
"expected_log_exceptions": [
launch_objects.LogExceptionResult(logId=1,
foundExceptions=[])
]
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
es_client.es_client.scroll = MagicMock(return_value=json.loads(
utils.get_fixture(self.no_hits_search_rs)))
launches = [
launch_objects.Launch(**launch)
for launch in json.loads(test["index_rq"])
]
response = es_client.index_logs(launches)
test["has_errors"].should.equal(response.errors)
test["expected_count"].should.equal(response.took)
test["expected_log_exceptions"].should.equal(
response.logResults)
TestEsClient.shutdown_server(test["test_calls"])
def test_clean_index(self):
"""Test cleaning index logs"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search?scroll=5m&size=1000",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(
self.search_not_merged_logs_for_delete),
"rs":
utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rs": utils.get_fixture(self.delete_logs_rs),
},
{
"method": httpretty.GET,
"uri": "/1/_search?scroll=5m&size=1000",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_not_merged_logs),
"rs": utils.get_fixture(self.one_hit_search_rs),
},
{
"method": httpretty.POST,
"uri": "/_bulk?refresh=true",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.index_logs_rq),
"rs": utils.get_fixture(self.index_logs_rs),
},
],
"rq":
launch_objects.CleanIndex(ids=[1], project=1),
"expected_count":
1
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
},
],
"rq":
launch_objects.CleanIndex(ids=[1], project=2),
"expected_count":
0
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
es_client = esclient.EsClient(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
es_client.es_client.scroll = MagicMock(return_value=json.loads(
utils.get_fixture(self.no_hits_search_rs)))
response = es_client.delete_logs(test["rq"])
test["expected_count"].should.equal(response)
TestEsClient.shutdown_server(test["test_calls"])
def __test_something():
# same calculated value
with sure.ensure('neverused'):
raise Exception('This is not an AssertionError')
def step_impl(context):
do_build = os.path.join(context.fake_srcdir, 'do', 'build')
with sure.ensure('do/build should exist'):
os.path.exists(do_build).should.be.true
with sure.ensure('do/build should be executable'):
os.access(do_build, os.X_OK).should.be.true
def test_analyze_logs(self):
"""Test analyzing logs"""
tests = [
{
"test_calls": [{"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, ],
"index_rq": utils.get_fixture(self.launch_wo_test_items),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
}, ],
"index_rq": utils.get_fixture(
self.launch_w_test_items_wo_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, ],
"index_rq": utils.get_fixture(
self.launch_w_test_items_w_empty_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}, ],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.no_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.NOT_FOUND,
}, ],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.one_hit_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1], [[0.2, 0.8]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.one_hit_search_rs, to_json=True),
utils.get_fixture(self.two_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1, 1], [[0.2, 0.8], [0.3, 0.7]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "PB001",
"boost_predict": ([0, 1], [[0.8, 0.2], [0.3, 0.7]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1, 0], [[0.2, 0.8], [0.7, 0.3]])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [utils.get_fixture(self.two_hits_search_rs, to_json=True)],
"index_rq": utils.get_fixture(
self.launch_w_test_items_w_logs_to_be_merged),
"expected_count": 0,
"expected_issue_type": "",
"boost_predict": ([], [])
},
{
"test_calls": [{"method": httpretty.GET,
"uri": "/2",
"status": HTTPStatus.OK,
}],
"msearch_results": [
utils.get_fixture(self.no_hits_search_rs, to_json=True),
utils.get_fixture(self.three_hits_search_rs_with_one_unique_id, to_json=True)],
"index_rq": utils.get_fixture(
self.launch_w_test_items_w_logs),
"expected_count": 1,
"expected_issue_type": "AB001",
"boost_predict": ([1], [[0.2, 0.8]])
}
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
config = self.get_default_search_config()
analyzer_service = AutoAnalyzerService(app_config=self.app_config,
search_cfg=config)
_boosting_decision_maker = BoostingDecisionMaker()
_boosting_decision_maker.get_feature_ids = MagicMock(return_value=[0])
_boosting_decision_maker.predict = MagicMock(return_value=test["boost_predict"])
if "msearch_results" in test:
analyzer_service.es_client.es_client.msearch = MagicMock(
return_value={"responses": test["msearch_results"]})
analyzer_service.boosting_decision_maker = _boosting_decision_maker
launches = [launch_objects.Launch(**launch)
for launch in json.loads(test["index_rq"])]
response = analyzer_service.analyze_logs(launches)
response.should.have.length_of(test["expected_count"])
if test["expected_issue_type"] != "":
test["expected_issue_type"].should.equal(response[0].issueType)
if "expected_id" in test:
test["expected_id"].should.equal(response[0].relevantItem)
TestAutoAnalyzerService.shutdown_server(test["test_calls"])
def test_search_logs(self):
"""Test search logs"""
tests = [
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method": httpretty.GET,
"uri": "/1/_search",
"status": HTTPStatus.OK,
"content_type": "application/json",
"rq": utils.get_fixture(self.search_logs_rq),
"rs": utils.get_fixture(self.no_hits_search_rs),
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=["error"],
logLines=-1),
"expected_count":
0
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=[""],
logLines=-1),
"expected_count":
0
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq),
"rs":
utils.get_fixture(self.one_hit_search_rs_search_logs),
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=["error"],
logLines=-1),
"expected_count":
0
},
{
"test_calls": [
{
"method": httpretty.GET,
"uri": "/1",
"status": HTTPStatus.OK,
},
{
"method":
httpretty.GET,
"uri":
"/1/_search",
"status":
HTTPStatus.OK,
"content_type":
"application/json",
"rq":
utils.get_fixture(self.search_logs_rq_not_found),
"rs":
utils.get_fixture(self.two_hits_search_rs_search_logs),
},
],
"rq":
launch_objects.SearchLogs(launchId=1,
launchName="Launch 1",
itemId=3,
projectId=1,
filteredLaunchIds=[1],
logMessages=["error occured once"],
logLines=-1),
"expected_count":
1
},
]
for idx, test in enumerate(tests):
with sure.ensure('Error in the test case number: {0}', idx):
self._start_server(test["test_calls"])
search_service = SearchService(
app_config=self.app_config,
search_cfg=self.get_default_search_config())
response = search_service.search_logs(test["rq"])
response.should.have.length_of(test["expected_count"])
TestSearchService.shutdown_server(test["test_calls"])
