def test_observer_content_aware_subjective_model_subjreject(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) with self.assertRaises(AssertionError): subjective_model.run_modeling(subject_rejection=True)
def test_observer_content_aware_subjective_model_nosubject(self): subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file( self.dataset_filepath) result = subjective_model.run_modeling(mode='NO_SUBJECT') self.assertAlmostEquals(np.sum(result['observer_bias']), 0.0, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.0, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 0.0, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.0, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.0384615384616, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4012220200639218, places=4) self.assertAlmostEquals(np.sum(result['content_bias']), 0.0, places=4) self.assertAlmostEquals(np.var(result['content_bias']), 0.0, places=4) self.assertAlmostEquals(np.sum(result['content_ambiguity']), 6.06982228334157, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0045809756997836721, places=4)
def test_observer_content_aware_subjective_model_nocontent(self): subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file( self.dataset_filepath) result = subjective_model.run_modeling(mode='NO_CONTENT') self.assertAlmostEquals(np.sum(result['observer_bias']), -0.090840910829083799, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.089032585621095089, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 15.681766163430936, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.012565584832977776, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.31447815213642, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4355485462027884, places=4) self.assertAlmostEquals(np.sum(result['content_bias']), 0.0, places=4) self.assertAlmostEquals(np.var(result['content_bias']), 0.0, places=4) self.assertAlmostEquals(np.sum(result['content_ambiguity']), 0.0, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0, places=4)
def test_observer_content_aware_subjective_model(self): subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file( self.dataset_filepath) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_bias']), 0, places=4) self.assertAlmostEquals(np.var(result['content_bias']), 0, places=4) self.assertAlmostEquals(np.sum(result['content_ambiguity']), 3.8972884776604402, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0041122094732031289, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), -0.055712761348815837, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.085842891905121704, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 10.164665557559516, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.028749990587721687, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.20774261173619, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4351342153719635, places=4)
def test_observer_content_aware_subjective_model_missingdata(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'missing_probability': 0.1, } dataset_reader = MissingDataRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_ambiguity']), 3.9104244772977128, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0037713583509767193, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), -0.21903272050455846, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.084353684687185043, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 9.8168943054654481, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.028159236075789944, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.05548186797336, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4339487982797514, places=4) np.random.seed(0) info_dict = { 'missing_probability': 0.5, } dataset_reader = MissingDataRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_ambiguity']), 2.63184284168883, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.019164097909450246, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), 0.2263148440748638, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.070613033112114504, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 12.317917502439435, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.029455722248727296, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.29962156788139, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4717366222424826, places=4)
def test_observer_content_aware_subjective_model_numerical(self): subjective_model = MaximumLikelihoodEstimationModel.from_dataset_file( self.dataset_filepath) result = subjective_model.run_modeling(gradient_method='numerical') self.assertAlmostEquals(np.sum(result['content_ambiguity']), 3.8972884776604402, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0041122094732031289, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), -0.055712761348815837, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.085842891905121704, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 10.164665557559516, places=3) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.028749990587721687, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.20774261173619, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4351342153719635, places=4) self.assertAlmostEquals(np.sum(result['content_ambiguity_std']), 0.30465244947706538, places=4) self.assertAlmostEquals(np.sum(result['observer_bias_std']), 1.7392847550878989, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency_std']), 12.393285044624955, places=4) self.assertAlmostEquals(np.sum(result['quality_scores_std']), 8.8863877635750423, places=4)
def test_observer_content_aware_subjective_model_missingdata(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'missing_probability': 0.1, } dataset_reader = MissingDataRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_bias']), 0, places=4) self.assertAlmostEquals(np.var(result['content_bias']), 0, places=4) self.assertAlmostEquals(np.sum(result['content_ambiguity']), 3.9104244772977128, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0037713583509767193, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), -0.21903272050455846, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.084353684687185043, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 9.8168943054654481, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.028159236075789944, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.05548186797336, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4339487982797514, places=4) np.random.seed(0) info_dict = { 'missing_probability': 0.5, } dataset_reader = MissingDataRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_bias']), 0, places=4) self.assertAlmostEquals(np.var(result['content_bias']), 0, places=4) self.assertAlmostEquals(np.sum(result['content_ambiguity']), 2.63184284168883, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.019164097909450246, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), 0.2263148440748638, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.070613033112114504, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 12.317917502439435, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.029455722248727296, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.29962156788139, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4717366222424826, places=4)