def test_compute_true_score_evaluations_sparse(self): expected_df = pd.DataFrame({'N': 10000, "N raters": 4, "N single": 3421, "N multiple": 6579, "Variance of errors": 0.5150882, "True score var": 0.769816, 'MSE true': 0.3550792, 'PRMSE true': 0.538748}, index=['system']) df_prmse = get_true_score_evaluations(self.data_sparse, self.system_score_columns, self.human_score_columns) assert_frame_equal(df_prmse, expected_df, check_dtype=False)
def test_compute_true_score_evaluations_full(self): expected_df = pd.DataFrame({'N': 10000, "N raters": 4, "N single": 0, "N multiple": 10000, "Variance of errors": 0.509375, "True score var": 0.7765515, 'MSE true': 0.3564625, 'PRMSE true': 0.5409673}, index=['system']) df_prmse = get_true_score_evaluations(self.data_full, self.system_score_columns, self.human_score_columns) assert_frame_equal(df_prmse, expected_df, check_dtype=False)
def test_get_true_score_evaluations_single_human_no_ve(): df = pd.DataFrame({'system': [1, 2, 5], 'sc1': [2, 3, 5]}) get_true_score_evaluations(df, 'system', 'sc1')