예제 #1
0
    def setUp(self):
        dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py'
        dataset = import_python_file(dataset_filepath)

        np.random.seed(0)
        info_dict = {
            'quality_scores': np.random.randint(1, 6, 79),
            'observer_bias': np.random.normal(0, 1, 26),
            'observer_inconsistency': np.abs(np.random.normal(0, 0.1, 26)),
            'content_bias': np.zeros(9),
            'content_ambiguity': np.zeros(9),
        }

        self.dataset_reader = SyntheticRawDatasetReader(dataset, input_dict=info_dict)
예제 #2
0
class SyntheticDatasetReaderTest(unittest.TestCase):
    def setUp(self):
        dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py'
        dataset = import_python_file(dataset_filepath)

        np.random.seed(0)
        info_dict = {
            'quality_scores': np.random.randint(1, 6, 79),
            'observer_bias': np.random.normal(0, 1, 26),
            'observer_inconsistency': np.abs(np.random.normal(0, 0.1, 26)),
            'content_bias': np.zeros(9),
            'content_ambiguity': np.zeros(9),
        }

        self.dataset_reader = SyntheticRawDatasetReader(dataset,
                                                        input_dict=info_dict)

    def test_read_dataset_stats(self):
        self.assertEquals(self.dataset_reader.num_ref_videos, 9)
        self.assertEquals(self.dataset_reader.num_dis_videos, 79)
        self.assertEquals(self.dataset_reader.num_observers, 26)

    def test_opinion_score_2darray(self):
        os_2darray = self.dataset_reader.opinion_score_2darray
        self.assertAlmostEquals(np.mean(os_2darray),
                                3.1912209428772669,
                                places=4)

    def test_dis_videos_content_ids(self):
        content_ids = self.dataset_reader.content_id_of_dis_videos
        self.assertAlmostEquals(np.mean(content_ids),
                                3.8607594936708862,
                                places=4)

    def test_disvideo_is_refvideo(self):
        l = self.dataset_reader.disvideo_is_refvideo
        self.assertItemsEqual(indices(l, lambda e: e is True), range(9))

    def test_ref_score(self):
        self.assertEqual(self.dataset_reader.ref_score, 5.0)

    def test_to_dataset(self):
        dataset = self.dataset_reader.to_dataset()

        old_scores = [
            dis_video['os']
            for dis_video in self.dataset_reader.dataset.dis_videos
        ]
        new_scores = [dis_video['os'] for dis_video in dataset.dis_videos]

        self.assertNotEquals(old_scores, new_scores)
예제 #3
0
class SyntheticDatasetReaderTest(unittest.TestCase):

    def setUp(self):
        dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py'
        dataset = import_python_file(dataset_filepath)

        np.random.seed(0)
        info_dict = {
            'quality_scores': np.random.randint(1, 6, 79),
            'observer_bias': np.random.normal(0, 1, 26),
            'observer_inconsistency': np.abs(np.random.normal(0, 0.1, 26)),
            'content_bias': np.zeros(9),
            'content_ambiguity': np.zeros(9),
        }

        self.dataset_reader = SyntheticRawDatasetReader(dataset, input_dict=info_dict)

    def test_read_dataset_stats(self):
        self.assertEquals(self.dataset_reader.num_ref_videos, 9)
        self.assertEquals(self.dataset_reader.num_dis_videos, 79)
        self.assertEquals(self.dataset_reader.num_observers, 26)

    def test_opinion_score_2darray(self):
        os_2darray = self.dataset_reader.opinion_score_2darray
        self.assertAlmostEquals(np.mean(os_2darray), 3.1912209428772669, places=4)

    def test_dis_videos_content_ids(self):
        content_ids = self.dataset_reader.content_id_of_dis_videos
        self.assertAlmostEquals(np.mean(content_ids), 3.8607594936708862, places=4)

    def test_disvideo_is_refvideo(self):
        l = self.dataset_reader.disvideo_is_refvideo
        self.assertItemsEqual(indices(l, lambda e: e is True), range(9))

    def test_ref_score(self):
        self.assertEqual(self.dataset_reader.ref_score, 5.0)

    def test_to_dataset(self):
        dataset = self.dataset_reader.to_dataset()

        old_scores = [dis_video['os'] for dis_video in self.dataset_reader.dataset.dis_videos]
        new_scores = [dis_video['os'] for dis_video in dataset.dis_videos]

        self.assertNotEquals(old_scores, new_scores)
예제 #4
0
    def test_observer_aware_subjective_model_synthetic(self):

        np.random.seed(0)

        dataset = import_python_file(self.dataset_filepath)
        info_dict = {
            'quality_scores': np.random.uniform(1, 5, 79),
            'observer_bias': np.random.normal(0, 1, 26),
            'observer_inconsistency': np.abs(np.random.uniform(0.4, 0.6, 26)),
            'content_bias': np.zeros(9),
            'content_ambiguity': np.zeros(9),
        }
        dataset_reader = SyntheticRawDatasetReader(dataset,
                                                   input_dict=info_dict)
        subjective_model = MaximumLikelihoodEstimationModelReduced(
            dataset_reader)
        result = subjective_model.run_modeling()

        self.assertAlmostEquals(np.sum(result['observer_bias']),
                                -0.90138622499935517,
                                places=4)
        self.assertAlmostEquals(np.var(result['observer_bias']),
                                0.84819162765420342,
                                places=4)

        self.assertAlmostEquals(np.sum(result['observer_inconsistency']),
                                12.742288471632817,
                                places=4)
        self.assertAlmostEquals(np.var(result['observer_inconsistency']),
                                0.0047638169604076975,
                                places=4)

        self.assertAlmostEquals(np.sum(result['quality_scores']),
                                236.78529213581052,
                                places=4)
        self.assertAlmostEquals(np.var(result['quality_scores']),
                                1.3059726132293354,
                                places=4)
예제 #5
0
def _validate_with_synthetic_dataset(subjective_model_classes,
                                     dataset_filepath, synthetic_result):

    dataset = import_python_file(dataset_filepath)

    dataset_reader = SyntheticRawDatasetReader(dataset,
                                               input_dict=synthetic_result)

    subjective_models = map(
        lambda subjective_model_class: subjective_model_class(dataset_reader),
        subjective_model_classes)

    results = map(lambda subjective_model: subjective_model.run_modeling(),
                  subjective_models)

    # ===== plot scatter =====
    fig, axs = plt.subplots(figsize=(9, 9), nrows=2, ncols=2)
    for subjective_model_class, result, idx in zip(subjective_model_classes,
                                                   results,
                                                   range(len(results))):

        model_name = subjective_model_class.TYPE

        ax = axs.item(0)
        if 'quality_scores' in result and 'quality_scores' in synthetic_result:
            color = color_dict[
                model_name] if model_name in color_dict else 'black'
            x = synthetic_result['quality_scores']
            y = result['quality_scores']
            ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))

        ax = axs.item(1)
        if 'observer_bias' in result and 'observer_bias' in synthetic_result:
            color = color_dict[
                model_name] if model_name in color_dict else 'black'
            x = synthetic_result['observer_bias']
            y = result['observer_bias']
            min_xy = np.min([len(x), len(y)])
            x = x[:min_xy]
            y = y[:min_xy]
            ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))

        ax = axs.item(2)
        if 'observer_inconsistency' in result and 'observer_inconsistency' in synthetic_result:
            color = color_dict[
                model_name] if model_name in color_dict else 'black'
            x = synthetic_result['observer_inconsistency']
            y = result['observer_inconsistency']
            min_xy = np.min([len(x), len(y)])
            x = x[:min_xy]
            y = y[:min_xy]
            ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))

        ax = axs.item(3)
        if 'content_ambiguity' in result and 'content_ambiguity' in synthetic_result:
            color = color_dict[
                model_name] if model_name in color_dict else 'black'
            x = synthetic_result['content_ambiguity']
            y = result['content_ambiguity']
            ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))

    axs.item(0).set_title(r'Quality Score ($x_e$)')
    axs.item(1).set_title(r'Subject Bias ($b_s$)')
    axs.item(2).set_title(r'Subject Inconsisency ($v_s$)')
    # axs.item(3).set_title(r'Content Bias ($\mu_c$)')
    axs.item(3).set_title(r'Content Ambiguity ($a_c$)')

    for i in range(4):
        ax = axs.item(i)
        ax.set_xlabel('Synthetic')
        ax.set_ylabel('Recovered')
        ax.grid()

    plt.tight_layout()