def test_make_dist_mat(self): """ Tests MaScorer.make_dist_mat :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.make_dist_mat(test_warnings, test_gsr) expected = np.array([156.672, 156.672, 22.173, 22.173]).reshape(4, 1) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_gsr.append(test_warnings[-1]) expected = np.array( [156.672, 145.956, 156.672, 145.956, 22.173, 0, 22.173, 0]).reshape(4, 2) result = MaScorer.make_dist_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) dist_mat_filename = "test_lb_dist_matrix.csv" dist_mat_path = os.path.join(LB_MA_TEST_PATH, dist_mat_filename) expected = np.genfromtxt(dist_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_dist_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res)
def test_make_ls_mat(self): """ Tests MaScorer.make_ls_mat :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) expected = np.array([0, 0, 0.778, 0.778]).reshape(4, 1) result = MaScorer.make_ls_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_gsr.append(test_warnings[-1]) test_gsr[-1]["Approximate_Location"] = "False" expected = np.array([0, 0, 0, 0, 0.778, 1, 0.778, 1]).reshape(4, 2) result = MaScorer.make_ls_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) ls_mat_filename = "test_ls_matrix.csv" ls_mat_path = os.path.join(LB_MA_TEST_PATH, ls_mat_filename) expected = np.genfromtxt(ls_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_ls_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res)
def test_make_as_mat(self): """ Tests MaScorer.make_as_mat :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) acs_mat_filename = "test_actor_match_matrix.csv" acs_mat_path = os.path.join(LB_MA_TEST_PATH, acs_mat_filename) expected = np.genfromtxt(acs_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_as_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res)
def test_score(self): """ Tests MaScorer.score method :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.score(test_warnings, test_gsr) expected_filename = "match_results.json" path_ = os.path.join(LB_MA_TEST_PATH, expected_filename) with open(path_, "r", encoding="utf8") as f: expected = json.load(f) expected_matches = sorted( set([(m["Warning"], m["Event"]) for m in expected["Matches"]])) expected_qs_ser = expected["Details"]["Quality Scores"] expected_qs_mean = expected["Quality Score"] expected_precision = expected["Precision"] expected_recall = expected["Recall"] expected_f1 = expected["F1"] self.assertEqual(sorted(set(result["Matches"])), expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], expected_precision, 3) self.assertAlmostEqual(result["Recall"], expected_recall, 3) self.assertAlmostEqual(result["F1"], expected_f1, 3) for i, qs in enumerate(expected_qs_ser): res_qs = result["Details"]["Quality Scores"][i] self.assertAlmostEqual(res_qs, qs, 3)
def test_subtype_score(self): """ Test MaScorer.event_subtype_score :return: """ warn_value = "Force Posture" gsr_value = "Conflict" expected = 0 result = MaScorer.event_subtype_score(warn_value, gsr_value) self.assertEqual(result, expected) gsr_value = "Force Posture" expected = 1 result = MaScorer.event_subtype_score(warn_value, gsr_value) self.assertEqual(result, expected) warn_value = "Sharpening Swords" expected = 0 result = MaScorer.event_subtype_score(warn_value, gsr_value) self.assertEqual(result, expected)
def test_duplicate_matches(self): """ Tests if the same GSR event or warning are matched multiply. :return: """ # Test using Lebanon for one participant. Known faiure case. test_warn_filename = "test_warn.json" test_warn_path = os.path.join(LB_PER1_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_gsr.json" test_gsr_path = os.path.join(LB_PER1_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.score(test_warnings, test_gsr) matches = result["Matches"] warn_ids = [m[0] for m in matches] warn_id_counter = Counter(warn_ids) max_warn_usage = warn_id_counter.most_common(1)[0][1] self.assertEqual(max_warn_usage, 1) gsr_ids = [m[1] for m in matches] gsr_id_counter = Counter(gsr_ids) max_gsr_usage = gsr_id_counter.most_common(1)[0][1] self.assertEqual(max_gsr_usage, 1) # Test using Syria, May 2018. This will take a while to run. test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(SY_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(SY_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.score(test_warnings, test_gsr) matches = result["Matches"] warn_ids = [m[0] for m in matches] warn_id_counter = Counter(warn_ids) max_warn_usage = warn_id_counter.most_common(1)[0][1] self.assertEqual(max_warn_usage, 1) gsr_ids = [m[1] for m in matches] gsr_id_counter = Counter(gsr_ids) max_gsr_usage = gsr_id_counter.most_common(1)[0][1] self.assertEqual(max_gsr_usage, 1)
def test_actor_score(self): """ Test MaScorer.actor_score :return: """ wildcards = ["Unspecified", "Wildcard"] legits = ["Fred", "Ethel"] warn_value = "Fred" gsr_value = "Ethel" expected = 0 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) gsr_value = wildcards[0] expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) gsr_value = wildcards[1] expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = wildcards[0] gsr_value = "Ethel" expected = 0 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = "Fred" gsr_value = ["Ethel", "Fred"] expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = "Lucy" gsr_value = ["Ethel", "Fred"] expected = 0 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected)
def test_ls(self): result = MaScorer.location_score(0, is_approximate=False) expected = 1.0 self.assertAlmostEqual(result, expected) result = MaScorer.location_score(0, is_approximate="False") expected = 1.0 self.assertAlmostEqual(result, expected) # 22 km distance result = MaScorer.location_score(22) expected = 0.78 self.assertAlmostEqual(result, expected, 2) result = MaScorer.location_score(22.17, is_approximate=True) expected = 0.934 self.assertAlmostEqual(result, expected, 3) result = MaScorer.location_score(22.1, max_dist=44.2) expected = 0.50 self.assertAlmostEqual(result, expected, 2) result = MaScorer.location_score(150) expected = 0.0 self.assertAlmostEqual(result, expected)
def test_score_one(self): """ Test MaScorer.score_one weights input :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) #print(test_gsr[0]) LEGIT_ACTORS = ["Egyptian Police"] bad_weight = -1 sub_weight = .5 super_weight = 2 # Test with default weights # Test with bad weights result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ls_weight=bad_weight) self.assertTrue("Errors" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ds_weight=bad_weight) self.assertTrue("Errors" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, as_weight=bad_weight) self.assertTrue("Errors" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ess_weight=bad_weight) self.assertTrue("Errors" in result) # Test with weights summing to less than 4 result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ls_weight=sub_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ds_weight=sub_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, as_weight=sub_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ess_weight=sub_weight) self.assertTrue("Notices" in result) # Test with weights summing to more than 4 result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ls_weight=super_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ds_weight=super_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, as_weight=super_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ess_weight=super_weight) self.assertTrue("Notices" in result) print("Result using default weights") result = MaScorer.score_one(test_warnings[3], test_gsr[0]) print(result) # Test a warning with LS = 0 result = MaScorer.score_one(test_warnings[1], test_gsr[0]) self.assertEqual(result[ScoreComponents.QS], 0) self.assertEqual(result[ScoreComponents.LS], 0) self.assertAlmostEqual(result[ScoreComponents.DS], 0.75) # Test a warning with DS = 0 result = MaScorer.score_one(test_warnings[2], test_gsr[0]) self.assertEqual(result[ScoreComponents.QS], 0) self.assertAlmostEqual(result[ScoreComponents.LS], 0.778, 3) self.assertEqual(result[ScoreComponents.DS], 0) # Test a legitimately matched warning result = MaScorer.score_one(test_warnings[3], test_gsr[0], legit_actors=LEGIT_ACTORS) self.assertEqual(result[ScoreComponents.AS], 1) self.assertEqual(result[ScoreComponents.ESS], 1) self.assertAlmostEqual(result[ScoreComponents.LS], 0.778, 3) self.assertAlmostEqual(result[ScoreComponents.DS], 0.75) self.assertAlmostEqual(result[ScoreComponents.QS], 0.882, 3) self.assertFalse("Notices" in result) self.assertFalse("Errors" in result)
def test_match(self): """ Tests MaScorer.match :return: """ # Simple Matrix, 3 by 4 test_matrix_filename = "test_qs_matrix_1.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [("warn_0", "evt_0"), ("warn_1", "evt_1"), ("warn_2", "evt_3")] expected_qs_ser = [1, 0.85, 0.8] expected_qs_mean = np.mean(expected_qs_ser) result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 1.0) self.assertAlmostEqual(result["Recall"], 0.75) self.assertAlmostEqual(result["F1"], 1.5 / 1.75) self.assertAlmostEqual(result["Mercury Score"], (expected_qs_mean + 1.5 / 1.75) / 2.) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Simple matrix, 4 by 3 test_matrix_filename = "test_qs_matrix_2.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [("warn_0", "evt_0"), ("warn_1", "evt_1"), ("warn_3", "evt_2")] expected_qs_ser = [1, 0.85, 0.75] expected_qs_mean = np.mean(expected_qs_ser) result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 0.75) self.assertAlmostEqual(result["Recall"], 1.00) self.assertAlmostEqual(result["F1"], 1.5 / 1.75) self.assertAlmostEqual(result["Mercury Score"], (expected_qs_mean + 1.5 / 1.75) / 2.) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Null Matrix test_matrix_filename = "test_null_matrix.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [] expected_qs_ser = [] expected_qs_mean = 0 result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 0) self.assertAlmostEqual(result["Recall"], 0) self.assertAlmostEqual(result["F1"], 0) self.assertAlmostEqual(result["Mercury Score"], 0) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Matrix with negative entries test_matrix_filename = "test_neg_matrix.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [("warn_0", "evt_0"), ("warn_2", "evt_2")] expected_qs_ser = [0.75, 1.0] expected_qs_mean = 0.875 result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 0.5) self.assertAlmostEqual(result["Recall"], 0.667, 3) self.assertAlmostEqual(result["F1"], 0.667 / 1.167, 3) self.assertAlmostEqual(result["Mercury Score"], (expected_qs_mean + 0.667 / 1.167) / 2, 3) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Matrix with Lebanon data test_matrix_filename = "test_qs_mat.csv" path_ = os.path.join(LB_MA_TEST_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_filename = "match_results.json" path_ = os.path.join(LB_MA_TEST_PATH, expected_filename) with open(path_, "r", encoding="utf8") as f: expected = json.load(f) expected_matches = sorted( set([(m["Warning"], m["Event"]) for m in expected["Matches"]])) expected_qs_ser = expected["Details"]["Quality Scores"] expected_qs_mean = expected["Quality Score"] expected_precision = expected["Precision"] expected_recall = expected["Recall"] expected_f1 = expected["F1"] result = MaScorer.match(input_matrix=test_mat) self.assertEqual(sorted(set(result["Matches"])), expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], expected_precision, 3) self.assertAlmostEqual(result["Recall"], expected_recall, 3) self.assertAlmostEqual(result["F1"], expected_f1, 3) self.assertAlmostEqual(result["Mercury Score"], (expected["Quality Score"] + expected["F1"]) / 2, 3) for i, qs in enumerate(expected_qs_ser): res_qs = result["Details"]["Quality Scores"][i] self.assertAlmostEqual(res_qs, qs, 3)
def test_make_qs_mat(self): """ Tests MaScorer.make_qs_df :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(LB_MA_TEST_PATH, mat_filename) expected = pd.read_csv(mat_path, index_col=0) result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(EG_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(EG_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(EG_MA_TEST_PATH, mat_filename) expected = np.genfromtxt(mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(SA_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(SA_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(SA_MA_TEST_PATH, mat_filename) expected = np.genfromtxt(mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(IQ_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(IQ_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(IQ_MA_TEST_PATH, mat_filename) expected = np.genfromtxt(mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res)
class MaScorerTest(unittest.TestCase): country = "Egypt" scorer = MaScorer(country=country) warn_dict = dict() warn_dict[JSONField.WARNING_ID] = "test_1" warn_dict[JSONField.EVENT_TYPE] = "Military Action" warn_dict[JSONField.COUNTRY] = country warn_dict[JSONField.EVENT_DATE] = "2018-05-27" warn_dict[JSONField.TIMESTAMP] = "20160324T00:01:01" gsr_dict = dict() gsr_dict[JSONField.EVENT_TYPE] = "Disease" gsr_dict[JSONField.EVENT_ID] = "Disease_Saudi_Arabia_MERS_2016-03-27" gsr_dict[JSONField.DISEASE] = "MERS" gsr_dict[JSONField.COUNTRY] = "Saudi Arabia" gsr_dict[JSONField.EVENT_DATE] = "2016-03-27" gsr_dict[JSONField.EARLIEST_REPORTED_DATE] = "2016-04-01" result_dict = dict() result_dict[JSONField.WARNING_ID] = "test_1" result_dict[JSONField.EVENT_ID] = "Disease_Saudi_Arabia_MERS_2016-03-27" def test_ls(self): result = MaScorer.location_score(0, is_approximate=False) expected = 1.0 self.assertAlmostEqual(result, expected) result = MaScorer.location_score(0, is_approximate="False") expected = 1.0 self.assertAlmostEqual(result, expected) # 22 km distance result = MaScorer.location_score(22) expected = 0.78 self.assertAlmostEqual(result, expected, 2) result = MaScorer.location_score(22.17, is_approximate=True) expected = 0.934 self.assertAlmostEqual(result, expected, 3) result = MaScorer.location_score(22.1, max_dist=44.2) expected = 0.50 self.assertAlmostEqual(result, expected, 2) result = MaScorer.location_score(150) expected = 0.0 self.assertAlmostEqual(result, expected) def test_make_dist_mat(self): """ Tests MaScorer.make_dist_mat :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.make_dist_mat(test_warnings, test_gsr) expected = np.array([156.672, 156.672, 22.173, 22.173]).reshape(4, 1) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_gsr.append(test_warnings[-1]) expected = np.array( [156.672, 145.956, 156.672, 145.956, 22.173, 0, 22.173, 0]).reshape(4, 2) result = MaScorer.make_dist_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) dist_mat_filename = "test_lb_dist_matrix.csv" dist_mat_path = os.path.join(LB_MA_TEST_PATH, dist_mat_filename) expected = np.genfromtxt(dist_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_dist_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) def test_make_ls_mat(self): """ Tests MaScorer.make_ls_mat :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) expected = np.array([0, 0, 0.778, 0.778]).reshape(4, 1) result = MaScorer.make_ls_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_gsr.append(test_warnings[-1]) test_gsr[-1]["Approximate_Location"] = "False" expected = np.array([0, 0, 0, 0, 0.778, 1, 0.778, 1]).reshape(4, 2) result = MaScorer.make_ls_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) ls_mat_filename = "test_ls_matrix.csv" ls_mat_path = os.path.join(LB_MA_TEST_PATH, ls_mat_filename) expected = np.genfromtxt(ls_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_ls_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) def test_make_ds_mat(self): """ Tests MaScorer.make_ds_mat :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) expected = np.array([0, .75, 0, .75]).reshape(4, 1) result = MaScorer.make_ds_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) ds_mat_filename = "test_ds_matrix.csv" ds_mat_path = os.path.join(LB_MA_TEST_PATH, ds_mat_filename) expected = np.genfromtxt(ds_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_ds_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) def test_make_ess_mat(self): """ Tests MaScorer.make_ess_mat :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) ess_mat_filename = "test_es_match_matrix.csv" ess_mat_path = os.path.join(LB_MA_TEST_PATH, ess_mat_filename) expected = np.genfromtxt(ess_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_ess_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) def test_make_as_mat(self): """ Tests MaScorer.make_as_mat :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) acs_mat_filename = "test_actor_match_matrix.csv" acs_mat_path = os.path.join(LB_MA_TEST_PATH, acs_mat_filename) expected = np.genfromtxt(acs_mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_as_mat(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) def test_make_qs_mat(self): """ Tests MaScorer.make_qs_df :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(LB_MA_TEST_PATH, mat_filename) expected = pd.read_csv(mat_path, index_col=0) result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(EG_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(EG_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(EG_MA_TEST_PATH, mat_filename) expected = np.genfromtxt(mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(SA_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(SA_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(SA_MA_TEST_PATH, mat_filename) expected = np.genfromtxt(mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(IQ_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(IQ_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) mat_filename = "test_qs_mat.csv" mat_path = os.path.join(IQ_MA_TEST_PATH, mat_filename) expected = np.genfromtxt(mat_path, delimiter=",", skip_header=True)[:, 1:] result = MaScorer.make_qs_df(test_warnings, test_gsr) try: np.testing.assert_allclose(result, expected, 3) test_res = True except AssertionError as e: test_res = False print(repr(e)) self.assertTrue(test_res) def test_facet_score(self): """ Tests Scorer.facet_score :return: """ wildcards = ["Unspecified", "Wildcard"] warn_value = "Fred" gsr_value = "Ethel" expected = 0 result = Scorer.facet_score(warn_value, gsr_value, wildcards) self.assertEqual(result, expected) gsr_value = wildcards[0] expected = 1 result = Scorer.facet_score(warn_value, gsr_value, wildcards) self.assertEqual(result, expected) gsr_value = wildcards[1] expected = 1 result = Scorer.facet_score(warn_value, gsr_value, wildcards) self.assertEqual(result, expected) warn_value = wildcards[0] gsr_value = "Ethel" expected = 0 result = Scorer.facet_score(warn_value, gsr_value, wildcards) self.assertEqual(result, expected) warn_value = "Fred" gsr_value = ["Ethel", "Fred"] expected = 1 result = Scorer.facet_score(warn_value, gsr_value, wildcards) self.assertEqual(result, expected) gsr_value = "Ethel;Fred" expected = 1 result = Scorer.facet_score(warn_value, gsr_value, wildcards) self.assertEqual(result, expected) gsr_value = "Ethel,Fred" expected = 1 result = Scorer.facet_score(warn_value, gsr_value, wildcards, gsr_value_delim=",") self.assertEqual(result, expected) def test_actor_score(self): """ Test MaScorer.actor_score :return: """ wildcards = ["Unspecified", "Wildcard"] legits = ["Fred", "Ethel"] warn_value = "Fred" gsr_value = "Ethel" expected = 0 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) gsr_value = wildcards[0] expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) gsr_value = wildcards[1] expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = wildcards[0] gsr_value = "Ethel" expected = 0 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = "Fred" gsr_value = ["Ethel", "Fred"] expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = "Fred" gsr_value = "Ethel;Fred" expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) warn_value = "Fred" gsr_value = "Ethel,Fred" expected = 1 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards, gsr_value_delim=",") self.assertEqual(result, expected) warn_value = "Lucy" gsr_value = ["Ethel", "Fred"] expected = 0 result = MaScorer.actor_score(warn_value, gsr_value, legits, wildcards) self.assertEqual(result, expected) def test_subtype_score(self): """ Test MaScorer.event_subtype_score :return: """ warn_value = "Force Posture" gsr_value = "Conflict" expected = 0 result = MaScorer.event_subtype_score(warn_value, gsr_value) self.assertEqual(result, expected) gsr_value = "Force Posture" expected = 1 result = MaScorer.event_subtype_score(warn_value, gsr_value) self.assertEqual(result, expected) warn_value = "Sharpening Swords" expected = 0 result = MaScorer.event_subtype_score(warn_value, gsr_value) self.assertEqual(result, expected) def test_score(self): """ Tests MaScorer.score method :return: """ test_warn_filename = "test_lb_warnings.json" test_warn_path = os.path.join(LB_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_lb_gsr.json" test_gsr_path = os.path.join(LB_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.score(test_warnings, test_gsr) expected_filename = "match_results.json" path_ = os.path.join(LB_MA_TEST_PATH, expected_filename) with open(path_, "r", encoding="utf8") as f: expected = json.load(f) expected_matches = sorted( set([(m["Warning"], m["Event"]) for m in expected["Matches"]])) expected_qs_ser = expected["Details"]["Quality Scores"] expected_qs_mean = expected["Quality Score"] expected_precision = expected["Precision"] expected_recall = expected["Recall"] expected_f1 = expected["F1"] expected_merc_score = expected["Mercury Score"] self.assertEqual(sorted(set(result["Matches"])), expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Mercury Score"], expected_merc_score, 3) self.assertAlmostEqual(result["Precision"], expected_precision, 3) self.assertAlmostEqual(result["Recall"], expected_recall, 3) self.assertAlmostEqual(result["F1"], expected_f1, 3) for i, qs in enumerate(expected_qs_ser): res_qs = result["Details"]["Quality Scores"][i] self.assertAlmostEqual(res_qs, qs, 3) def test_match(self): """ Tests MaScorer.match :return: """ # Simple Matrix, 3 by 4 test_matrix_filename = "test_qs_matrix_1.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [("warn_0", "evt_0"), ("warn_1", "evt_1"), ("warn_2", "evt_3")] expected_qs_ser = [1, 0.85, 0.8] expected_qs_mean = np.mean(expected_qs_ser) result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 1.0) self.assertAlmostEqual(result["Recall"], 0.75) self.assertAlmostEqual(result["F1"], 1.5 / 1.75) self.assertAlmostEqual(result["Mercury Score"], (expected_qs_mean + 1.5 / 1.75) / 2.) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Simple matrix, 4 by 3 test_matrix_filename = "test_qs_matrix_2.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [("warn_0", "evt_0"), ("warn_1", "evt_1"), ("warn_3", "evt_2")] expected_qs_ser = [1, 0.85, 0.75] expected_qs_mean = np.mean(expected_qs_ser) result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 0.75) self.assertAlmostEqual(result["Recall"], 1.00) self.assertAlmostEqual(result["F1"], 1.5 / 1.75) self.assertAlmostEqual(result["Mercury Score"], (expected_qs_mean + 1.5 / 1.75) / 2.) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Null Matrix test_matrix_filename = "test_null_matrix.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [] expected_qs_ser = [] expected_qs_mean = 0 result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 0) self.assertAlmostEqual(result["Recall"], 0) self.assertAlmostEqual(result["F1"], 0) self.assertAlmostEqual(result["Mercury Score"], 0) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Matrix with negative entries test_matrix_filename = "test_neg_matrix.csv" path_ = os.path.join(TEST_RESOURCE_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_matches = [("warn_0", "evt_0"), ("warn_2", "evt_2")] expected_qs_ser = [0.75, 1.0] expected_qs_mean = 0.875 result = MaScorer.match(input_matrix=test_mat) self.assertEqual(result["Matches"], expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], 0.5) self.assertAlmostEqual(result["Recall"], 0.667, 3) self.assertAlmostEqual(result["F1"], 0.667 / 1.167, 3) self.assertAlmostEqual(result["Mercury Score"], (expected_qs_mean + 0.667 / 1.167) / 2, 3) self.assertAlmostEqual(result["Details"]["Quality Scores"], expected_qs_ser, 3) # Matrix with Lebanon data test_matrix_filename = "test_qs_mat.csv" path_ = os.path.join(LB_MA_TEST_PATH, test_matrix_filename) test_mat = pd.read_csv(path_, index_col=0) expected_filename = "match_results.json" path_ = os.path.join(LB_MA_TEST_PATH, expected_filename) with open(path_, "r", encoding="utf8") as f: expected = json.load(f) expected_matches = sorted( set([(m["Warning"], m["Event"]) for m in expected["Matches"]])) expected_qs_ser = expected["Details"]["Quality Scores"] expected_qs_mean = expected["Quality Score"] expected_precision = expected["Precision"] expected_recall = expected["Recall"] expected_f1 = expected["F1"] result = MaScorer.match(input_matrix=test_mat) self.assertEqual(sorted(set(result["Matches"])), expected_matches) self.assertAlmostEqual(result["Quality Score"], expected_qs_mean, 3) self.assertAlmostEqual(result["Precision"], expected_precision, 3) self.assertAlmostEqual(result["Recall"], expected_recall, 3) self.assertAlmostEqual(result["F1"], expected_f1, 3) self.assertAlmostEqual(result["Mercury Score"], (expected["Quality Score"] + expected["F1"]) / 2, 3) for i, qs in enumerate(expected_qs_ser): res_qs = result["Details"]["Quality Scores"][i] self.assertAlmostEqual(res_qs, qs, 3) def test_score_one(self): """ Test MaScorer.score_one weights input :return: """ test_warn_filename = "ma_test_warnings.json" test_warn_path = os.path.join(TEST_RESOURCE_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "ma_test_gsr.json" test_gsr_path = os.path.join(TEST_RESOURCE_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) #print(test_gsr[0]) LEGIT_ACTORS = ["Egyptian Police"] bad_weight = -1 sub_weight = .5 super_weight = 2 # Test with default weights # Test with bad weights result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ls_weight=bad_weight) self.assertTrue("Errors" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ds_weight=bad_weight) self.assertTrue("Errors" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, as_weight=bad_weight) self.assertTrue("Errors" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ess_weight=bad_weight) self.assertTrue("Errors" in result) # Test with weights summing to less than 4 result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ls_weight=sub_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ds_weight=sub_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, as_weight=sub_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ess_weight=sub_weight) self.assertTrue("Notices" in result) # Test with weights summing to more than 4 result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ls_weight=super_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ds_weight=super_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, as_weight=super_weight) self.assertTrue("Notices" in result) result = MaScorer.score_one(test_warnings[0], test_gsr[0], legit_actors=LEGIT_ACTORS, ess_weight=super_weight) self.assertTrue("Notices" in result) print("Result using default weights") result = MaScorer.score_one(test_warnings[3], test_gsr[0]) print(result) # Test a warning with LS = 0 result = MaScorer.score_one(test_warnings[1], test_gsr[0]) self.assertEqual(result[ScoreComponents.QS], 0) self.assertEqual(result[ScoreComponents.LS], 0) self.assertAlmostEqual(result[ScoreComponents.DS], 0.75) # Test a warning with DS = 0 result = MaScorer.score_one(test_warnings[2], test_gsr[0]) self.assertEqual(result[ScoreComponents.QS], 0) self.assertAlmostEqual(result[ScoreComponents.LS], 0.778, 3) self.assertEqual(result[ScoreComponents.DS], 0) # Test a legitimately matched warning result = MaScorer.score_one(test_warnings[3], test_gsr[0], legit_actors=LEGIT_ACTORS) self.assertEqual(result[ScoreComponents.AS], 1) self.assertEqual(result[ScoreComponents.ESS], 1) self.assertAlmostEqual(result[ScoreComponents.LS], 0.778, 3) self.assertAlmostEqual(result[ScoreComponents.DS], 0.75) self.assertAlmostEqual(result[ScoreComponents.QS], 0.882, 3) self.assertFalse("Notices" in result) self.assertFalse("Errors" in result) def test_duplicate_matches(self): """ Tests if the same GSR event or warning are matched multiply. :return: """ # Test using Lebanon for one participant. Known faiure case. test_warn_filename = "test_warn.json" test_warn_path = os.path.join(LB_PER1_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_gsr.json" test_gsr_path = os.path.join(LB_PER1_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.score(test_warnings, test_gsr) matches = result["Matches"] warn_ids = [m[0] for m in matches] warn_id_counter = Counter(warn_ids) max_warn_usage = warn_id_counter.most_common(1)[0][1] self.assertEqual(max_warn_usage, 1) gsr_ids = [m[1] for m in matches] gsr_id_counter = Counter(gsr_ids) max_gsr_usage = gsr_id_counter.most_common(1)[0][1] self.assertEqual(max_gsr_usage, 1) # Test using Syria, May 2018. This will take a while to run. test_warn_filename = "test_cc_warnings.json" test_warn_path = os.path.join(SY_MA_TEST_PATH, test_warn_filename) with open(test_warn_path, "r", encoding="utf8") as f: test_warnings = json.load(f) test_gsr_filename = "test_cc_gsr.json" test_gsr_path = os.path.join(SY_MA_TEST_PATH, test_gsr_filename) with open(test_gsr_path, "r", encoding="utf8") as f: test_gsr = json.load(f) result = MaScorer.score(test_warnings, test_gsr) matches = result["Matches"] warn_ids = [m[0] for m in matches] warn_id_counter = Counter(warn_ids) max_warn_usage = warn_id_counter.most_common(1)[0][1] self.assertEqual(max_warn_usage, 1) gsr_ids = [m[1] for m in matches] gsr_id_counter = Counter(gsr_ids) max_gsr_usage = gsr_id_counter.most_common(1)[0][1] self.assertEqual(max_gsr_usage, 1)