def test_score(self):
"""
Tests MaScorer.score method
:return:
"""
test_warn_filename = "test_lb_warnings.json"
test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename)
with open(test_warn_path, "r", encoding="utf8") as f:
test_warnings = json.load(f)
test_gsr_filename = "test_lb_gsr.json"
test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename)
with open(test_gsr_path, "r", encoding="utf8") as f:
test_gsr = json.load(f)
result = MaScorer.score(test_warnings, test_gsr)
expected_filename = "match_results.json"
path_ = os.path.join(LB_MA_TEST_PATH, expected_filename)
with open(path_, "r", encoding="utf8") as f:
expected = json.load(f)
expected_matches = sorted(
set([(m["Warning"], m["Event"]) for m in expected["Matches"]]))
expected_qs_ser = expected["Details"]["Quality Scores"]
expected_qs_mean = expected["Quality Score"]
expected_precision = expected["Precision"]
expected_recall = expected["Recall"]
expected_f1 = expected["F1"]
self.assertEqual(sorted(set(result["Matches"])), expected_matches)
self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3)
self.assertAlmostEqual(result["Precision"], expected_precision, 3)
self.assertAlmostEqual(result["Recall"], expected_recall, 3)
self.assertAlmostEqual(result["F1"], expected_f1, 3)
for i, qs in enumerate(expected_qs_ser):
res_qs = result["Details"]["Quality Scores"][i]
self.assertAlmostEqual(res_qs, qs, 3)
def test_duplicate_matches(self):
"""
Tests if the same GSR event or warning are matched multiply.
:return:
"""
# Test using Lebanon for one participant. Known faiure case.
test_warn_filename = "test_warn.json"
test_warn_path = os.path.join(LB_PER1_TEST_PATH, test_warn_filename)
with open(test_warn_path, "r", encoding="utf8") as f:
test_warnings = json.load(f)
test_gsr_filename = "test_gsr.json"
test_gsr_path = os.path.join(LB_PER1_TEST_PATH, test_gsr_filename)
with open(test_gsr_path, "r", encoding="utf8") as f:
test_gsr = json.load(f)
result = MaScorer.score(test_warnings, test_gsr)
matches = result["Matches"]
warn_ids = [m[0] for m in matches]
warn_id_counter = Counter(warn_ids)
max_warn_usage = warn_id_counter.most_common(1)[0][1]
self.assertEqual(max_warn_usage, 1)
gsr_ids = [m[1] for m in matches]
gsr_id_counter = Counter(gsr_ids)
max_gsr_usage = gsr_id_counter.most_common(1)[0][1]
self.assertEqual(max_gsr_usage, 1)
# Test using Syria, May 2018. This will take a while to run.
test_warn_filename = "test_cc_warnings.json"
test_warn_path = os.path.join(SY_MA_TEST_PATH, test_warn_filename)
with open(test_warn_path, "r", encoding="utf8") as f:
test_warnings = json.load(f)
test_gsr_filename = "test_cc_gsr.json"
test_gsr_path = os.path.join(SY_MA_TEST_PATH, test_gsr_filename)
with open(test_gsr_path, "r", encoding="utf8") as f:
test_gsr = json.load(f)
result = MaScorer.score(test_warnings, test_gsr)
matches = result["Matches"]
warn_ids = [m[0] for m in matches]
warn_id_counter = Counter(warn_ids)
max_warn_usage = warn_id_counter.most_common(1)[0][1]
self.assertEqual(max_warn_usage, 1)
gsr_ids = [m[1] for m in matches]
gsr_id_counter = Counter(gsr_ids)
max_gsr_usage = gsr_id_counter.most_common(1)[0][1]
self.assertEqual(max_gsr_usage, 1)
