def setUp(self):
dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py'
dataset = import_python_file(dataset_filepath)
np.random.seed(0)
info_dict = {
'quality_scores': np.random.randint(1, 6, 79),
'observer_bias': np.random.normal(0, 1, 26),
'observer_inconsistency': np.abs(np.random.normal(0, 0.1, 26)),
'content_bias': np.zeros(9),
'content_ambiguity': np.zeros(9),
}
self.dataset_reader = SyntheticRawDatasetReader(dataset, input_dict=info_dict)
class SyntheticDatasetReaderTest(unittest.TestCase):
def setUp(self):
dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py'
dataset = import_python_file(dataset_filepath)
np.random.seed(0)
info_dict = {
'quality_scores': np.random.randint(1, 6, 79),
'observer_bias': np.random.normal(0, 1, 26),
'observer_inconsistency': np.abs(np.random.normal(0, 0.1, 26)),
'content_bias': np.zeros(9),
'content_ambiguity': np.zeros(9),
}
self.dataset_reader = SyntheticRawDatasetReader(dataset,
input_dict=info_dict)
def test_read_dataset_stats(self):
self.assertEquals(self.dataset_reader.num_ref_videos, 9)
self.assertEquals(self.dataset_reader.num_dis_videos, 79)
self.assertEquals(self.dataset_reader.num_observers, 26)
def test_opinion_score_2darray(self):
os_2darray = self.dataset_reader.opinion_score_2darray
self.assertAlmostEquals(np.mean(os_2darray),
3.1912209428772669,
places=4)
def test_dis_videos_content_ids(self):
content_ids = self.dataset_reader.content_id_of_dis_videos
self.assertAlmostEquals(np.mean(content_ids),
3.8607594936708862,
places=4)
def test_disvideo_is_refvideo(self):
l = self.dataset_reader.disvideo_is_refvideo
self.assertItemsEqual(indices(l, lambda e: e is True), range(9))
def test_ref_score(self):
self.assertEqual(self.dataset_reader.ref_score, 5.0)
def test_to_dataset(self):
dataset = self.dataset_reader.to_dataset()
old_scores = [
dis_video['os']
for dis_video in self.dataset_reader.dataset.dis_videos
]
new_scores = [dis_video['os'] for dis_video in dataset.dis_videos]
self.assertNotEquals(old_scores, new_scores)
class SyntheticDatasetReaderTest(unittest.TestCase):
def setUp(self):
dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py'
dataset = import_python_file(dataset_filepath)
np.random.seed(0)
info_dict = {
'quality_scores': np.random.randint(1, 6, 79),
'observer_bias': np.random.normal(0, 1, 26),
'observer_inconsistency': np.abs(np.random.normal(0, 0.1, 26)),
'content_bias': np.zeros(9),
'content_ambiguity': np.zeros(9),
}
self.dataset_reader = SyntheticRawDatasetReader(dataset, input_dict=info_dict)
def test_read_dataset_stats(self):
self.assertEquals(self.dataset_reader.num_ref_videos, 9)
self.assertEquals(self.dataset_reader.num_dis_videos, 79)
self.assertEquals(self.dataset_reader.num_observers, 26)
def test_opinion_score_2darray(self):
os_2darray = self.dataset_reader.opinion_score_2darray
self.assertAlmostEquals(np.mean(os_2darray), 3.1912209428772669, places=4)
def test_dis_videos_content_ids(self):
content_ids = self.dataset_reader.content_id_of_dis_videos
self.assertAlmostEquals(np.mean(content_ids), 3.8607594936708862, places=4)
def test_disvideo_is_refvideo(self):
l = self.dataset_reader.disvideo_is_refvideo
self.assertItemsEqual(indices(l, lambda e: e is True), range(9))
def test_ref_score(self):
self.assertEqual(self.dataset_reader.ref_score, 5.0)
def test_to_dataset(self):
dataset = self.dataset_reader.to_dataset()
old_scores = [dis_video['os'] for dis_video in self.dataset_reader.dataset.dis_videos]
new_scores = [dis_video['os'] for dis_video in dataset.dis_videos]
self.assertNotEquals(old_scores, new_scores)
def test_observer_aware_subjective_model_synthetic(self):
np.random.seed(0)
dataset = import_python_file(self.dataset_filepath)
info_dict = {
'quality_scores': np.random.uniform(1, 5, 79),
'observer_bias': np.random.normal(0, 1, 26),
'observer_inconsistency': np.abs(np.random.uniform(0.4, 0.6, 26)),
'content_bias': np.zeros(9),
'content_ambiguity': np.zeros(9),
}
dataset_reader = SyntheticRawDatasetReader(dataset,
input_dict=info_dict)
subjective_model = MaximumLikelihoodEstimationModelReduced(
dataset_reader)
result = subjective_model.run_modeling()
self.assertAlmostEquals(np.sum(result['observer_bias']),
-0.90138622499935517,
places=4)
self.assertAlmostEquals(np.var(result['observer_bias']),
0.84819162765420342,
places=4)
self.assertAlmostEquals(np.sum(result['observer_inconsistency']),
12.742288471632817,
places=4)
self.assertAlmostEquals(np.var(result['observer_inconsistency']),
0.0047638169604076975,
places=4)
self.assertAlmostEquals(np.sum(result['quality_scores']),
236.78529213581052,
places=4)
self.assertAlmostEquals(np.var(result['quality_scores']),
1.3059726132293354,
places=4)
def _validate_with_synthetic_dataset(subjective_model_classes,
dataset_filepath, synthetic_result):
dataset = import_python_file(dataset_filepath)
dataset_reader = SyntheticRawDatasetReader(dataset,
input_dict=synthetic_result)
subjective_models = map(
lambda subjective_model_class: subjective_model_class(dataset_reader),
subjective_model_classes)
results = map(lambda subjective_model: subjective_model.run_modeling(),
subjective_models)
# ===== plot scatter =====
fig, axs = plt.subplots(figsize=(9, 9), nrows=2, ncols=2)
for subjective_model_class, result, idx in zip(subjective_model_classes,
results,
range(len(results))):
model_name = subjective_model_class.TYPE
ax = axs.item(0)
if 'quality_scores' in result and 'quality_scores' in synthetic_result:
color = color_dict[
model_name] if model_name in color_dict else 'black'
x = synthetic_result['quality_scores']
y = result['quality_scores']
ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))
ax = axs.item(1)
if 'observer_bias' in result and 'observer_bias' in synthetic_result:
color = color_dict[
model_name] if model_name in color_dict else 'black'
x = synthetic_result['observer_bias']
y = result['observer_bias']
min_xy = np.min([len(x), len(y)])
x = x[:min_xy]
y = y[:min_xy]
ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))
ax = axs.item(2)
if 'observer_inconsistency' in result and 'observer_inconsistency' in synthetic_result:
color = color_dict[
model_name] if model_name in color_dict else 'black'
x = synthetic_result['observer_inconsistency']
y = result['observer_inconsistency']
min_xy = np.min([len(x), len(y)])
x = x[:min_xy]
y = y[:min_xy]
ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))
ax = axs.item(3)
if 'content_ambiguity' in result and 'content_ambiguity' in synthetic_result:
color = color_dict[
model_name] if model_name in color_dict else 'black'
x = synthetic_result['content_ambiguity']
y = result['content_ambiguity']
ax.scatter(x, y, color=color, label='{sm}'.format(sm=model_name))
axs.item(0).set_title(r'Quality Score ($x_e$)')
axs.item(1).set_title(r'Subject Bias ($b_s$)')
axs.item(2).set_title(r'Subject Inconsisency ($v_s$)')
# axs.item(3).set_title(r'Content Bias ($\mu_c$)')
axs.item(3).set_title(r'Content Ambiguity ($a_c$)')
for i in range(4):
ax = axs.item(i)
ax.set_xlabel('Synthetic')
ax.set_ylabel('Recovered')
ax.grid()
plt.tight_layout()