def test_observer_content_aware_subjective_model_nosubject(self):
subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file(
self.dataset_filepath)
result = subjective_model.run_modeling(mode='NO_SUBJECT')
self.assertAlmostEquals(np.sum(result['observer_bias']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['observer_bias']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']),
0.0,
places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']),
0.0,
places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']),
280.0384615384616,
places=4)
self.assertAlmostEquals(np.var(result['quality_scores']),
1.4012220200639218,
places=4)
self.assertAlmostEquals(np.sum(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['content_ambiguity']),
6.06982228334157,
places=4)
self.assertAlmostEquals(np.var(result['content_ambiguity']),
0.0045809756997836721,
places=4)
def test_observer_content_aware_subjective_model_nocontent(self):
subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file(
self.dataset_filepath)
result = subjective_model.run_modeling(mode='NO_CONTENT')
self.assertAlmostEquals(np.sum(result['observer_bias']),
-0.090840910829083799,
places=4)
self.assertAlmostEquals(np.var(result['observer_bias']),
0.089032585621095089,
places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']),
15.681766163430936,
places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']),
0.012565584832977776,
places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']),
280.31447815213642,
places=4)
self.assertAlmostEquals(np.var(result['quality_scores']),
1.4355485462027884,
places=4)
self.assertAlmostEquals(np.sum(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['content_ambiguity']),
0.0,
places=4)
self.assertAlmostEquals(np.var(result['content_ambiguity']),
0.0,
places=4)
def test_observer_content_aware_subjective_model(self):
subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file(
self.dataset_filepath)
result = subjective_model.run_modeling()
self.assertAlmostEquals(np.sum(result['content_bias']), 0, places=4)
self.assertAlmostEquals(np.var(result['content_bias']), 0, places=4)
self.assertAlmostEquals(np.sum(result['content_ambiguity']),
3.8972884776604402,
places=4)
self.assertAlmostEquals(np.var(result['content_ambiguity']),
0.0041122094732031289,
places=4)
self.assertAlmostEquals(np.sum(result['observer_bias']),
-0.055712761348815837,
places=4)
self.assertAlmostEquals(np.var(result['observer_bias']),
0.085842891905121704,
places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']),
10.164665557559516,
places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']),
0.028749990587721687,
places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']),
280.20774261173619,
places=4)
self.assertAlmostEquals(np.var(result['quality_scores']),
1.4351342153719635,
places=4)
def test_observer_content_aware_subjective_model_nosubject(self):
subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file(
self.dataset_filepath)
result = subjective_model.run_modeling(mode='NO_SUBJECT')
self.assertAlmostEquals(np.sum(result['observer_bias']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['observer_bias']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']), 280.0384615384616, places=4)
self.assertAlmostEquals(np.var(result['quality_scores']), 1.4012220200639218, places=4)
self.assertAlmostEquals(np.sum(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['content_ambiguity']), 6.06982228334157, places=4)
self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0045809756997836721, places=4)
def test_observer_content_aware_subjective_model_nocontent(self):
subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file(
self.dataset_filepath)
result = subjective_model.run_modeling(mode='NO_CONTENT')
self.assertAlmostEquals(np.sum(result['observer_bias']), -0.090840910829083799, places=4)
self.assertAlmostEquals(np.var(result['observer_bias']), 0.089032585621095089, places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 15.681766163430936, places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.012565584832977776, places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']), 280.31447815213642, places=4)
self.assertAlmostEquals(np.var(result['quality_scores']), 1.4355485462027884, places=4)
self.assertAlmostEquals(np.sum(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['content_bias']), 0.0, places=4)
self.assertAlmostEquals(np.sum(result['content_ambiguity']), 0.0, places=4)
self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0, places=4)
def test_observer_content_aware_subjective_model(self):
subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file(
self.dataset_filepath)
result = subjective_model.run_modeling()
self.assertAlmostEquals(np.sum(result['content_bias']), 0, places=4)
self.assertAlmostEquals(np.var(result['content_bias']), 0, places=4)
self.assertAlmostEquals(np.sum(result['content_ambiguity']), 3.8972884776604402, places=4)
self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0041122094732031289, places=4)
self.assertAlmostEquals(np.sum(result['observer_bias']), -0.055712761348815837, places=4)
self.assertAlmostEquals(np.var(result['observer_bias']), 0.085842891905121704, places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 10.164665557559516, places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.028749990587721687, places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']), 280.20774261173619, places=4)
self.assertAlmostEquals(np.var(result['quality_scores']), 1.4351342153719635, places=4)
