def test_compute_true_score_evaluations_sparse(self):
expected_df = pd.DataFrame({'N': 10000,
"N raters": 4,
"N single": 3421,
"N multiple": 6579,
"Variance of errors": 0.5150882,
"True score var": 0.769816,
'MSE true': 0.3550792,
'PRMSE true': 0.538748},
index=['system'])
df_prmse = get_true_score_evaluations(self.data_sparse,
self.system_score_columns,
self.human_score_columns)
assert_frame_equal(df_prmse, expected_df, check_dtype=False)
def test_compute_true_score_evaluations_full(self):
expected_df = pd.DataFrame({'N': 10000,
"N raters": 4,
"N single": 0,
"N multiple": 10000,
"Variance of errors": 0.509375,
"True score var": 0.7765515,
'MSE true': 0.3564625,
'PRMSE true': 0.5409673},
index=['system'])
df_prmse = get_true_score_evaluations(self.data_full,
self.system_score_columns,
self.human_score_columns)
assert_frame_equal(df_prmse, expected_df, check_dtype=False)
def test_get_true_score_evaluations_single_human_no_ve():
df = pd.DataFrame({'system': [1, 2, 5], 'sc1': [2, 3, 5]})
get_true_score_evaluations(df, 'system', 'sc1')