def test_train_test_on_raw_dataset_with_dis1st_thr(self): train_dataset = import_python_file( VmafConfig.test_resource_path('raw_dataset_sample.py')) model_param = import_python_file( VmafConfig.test_resource_path('model_param_sample.py')) feature_param = import_python_file( VmafConfig.test_resource_path('feature_param_sample.py')) train_fassembler, train_assets, train_stats, test_fassembler, test_assets, test_stats, _ = train_test_vmaf_on_dataset( train_dataset=train_dataset, test_dataset=train_dataset, feature_param=feature_param, model_param=model_param, train_ax=None, test_ax=None, result_store=None, parallelize=True, logger=None, fifo_mode=True, output_model_filepath=self.output_model_filepath ) self.train_fassembler = train_fassembler self.assertTrue(os.path.exists(self.output_model_filepath)) self.assertAlmostEqual(train_stats['ys_label_pred'][0], 93.565459224020742, places=3) self.assertAlmostEqual(test_stats['ys_label_pred'][0], 93.565459224020742, places=3)
def test_persubject_subjective_model_output(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = PerSubjectModel(dataset_reader) subjective_model.run_modeling(transform_final={'p1':25, 'p0':-25}) subjective_model.to_aggregated_dataset_file(self.output_dataset_filepath) self.assertTrue(os.path.exists(self.output_dataset_filepath)) dataset2 = import_python_file(self.output_dataset_filepath) dis_video = dataset2.dis_videos[0] self.assertTrue('groundtruth' in dis_video) self.assertTrue('os' not in dis_video) self.assertAlmostEquals(dis_video['groundtruth'], 100.0, places=4)
def test_mos_subjective_model_output(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) subjective_model.run_modeling() subjective_model.to_aggregated_dataset_file(self.output_dataset_filepath) self.assertTrue(os.path.exists(self.output_dataset_filepath)) dataset2 = import_python_file(self.output_dataset_filepath) dis_video = dataset2.dis_videos[0] self.assertTrue('groundtruth' in dis_video) self.assertTrue('os' not in dis_video) self.assertAlmostEquals(dis_video['groundtruth'], 4.884615384615385, places=4)
def test_mos_subjective_model_output_custom_resampling(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) subjective_model.run_modeling() subjective_model.to_aggregated_dataset_file(self.output_dataset_filepath, resampling_type='lanczos') self.assertTrue(os.path.exists(self.output_dataset_filepath)) dataset2 = import_python_file(self.output_dataset_filepath) self.assertFalse(hasattr(dataset2, 'quality_height')) self.assertFalse(hasattr(dataset2, 'quality_width')) self.assertEquals(dataset2.resampling_type, 'lanczos') dis_video = dataset2.dis_videos[0] self.assertTrue('groundtruth' in dis_video) self.assertTrue('os' not in dis_video) self.assertAlmostEquals(dis_video['groundtruth'], 4.884615384615385, places=4)
def test_read_dataset_mixed(self): dataset_path = VmafConfig.test_resource_path('test_dataset_mixed.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEquals(len(assets), 4) self.assertEqual(assets[0].resampling_type, 'bilinear') self.assertEqual(assets[0].ref_yuv_type, 'yuv420p') self.assertEqual(assets[0].dis_yuv_type, 'yuv420p') self.assertEqual(assets[0].ref_width_height, (1920, 1080)) self.assertEqual(assets[0].dis_width_height, (1920, 1080)) self.assertEqual(assets[1].resampling_type, 'bilinear') self.assertEqual(assets[1].ref_yuv_type, 'yuv420p') self.assertEqual(assets[1].dis_yuv_type, 'notyuv') self.assertEqual(assets[1].ref_width_height, (1920, 1080)) self.assertEqual(assets[1].dis_width_height, None) self.assertEqual(assets[2].resampling_type, 'bilinear') self.assertEqual(assets[2].ref_yuv_type, 'yuv420p') self.assertEqual(assets[2].dis_yuv_type, 'yuv420p') self.assertEqual(assets[2].ref_width_height, (720, 480)) self.assertEqual(assets[2].dis_width_height, (720, 480)) self.assertEqual(assets[3].resampling_type, 'bilinear') self.assertEqual(assets[3].ref_yuv_type, 'yuv420p') self.assertEqual(assets[3].dis_yuv_type, 'notyuv') self.assertEqual(assets[3].ref_width_height, (720, 480)) self.assertEqual(assets[3].dis_width_height, None)
def main(): if len(sys.argv) < 3: print_usage() return 2 try: quality_type = sys.argv[1] dataset_filepath = sys.argv[2] except ValueError: print_usage() return 2 try: dataset = import_python_file(dataset_filepath) except Exception as e: print "Error: " + str(e) return 1 try: runner_class = QualityRunner.find_subclass(quality_type) except: print_usage() return 2 result_store = FileSystemResultStore() run_remove_results_for_dataset(result_store, dataset, runner_class) return 0
def test_generate_dataset_from_raw_default(self): # DMOS generate_dataset_from_raw( raw_dataset_filepath=self.raw_dataset_filepath, output_dataset_filepath=self.derived_dataset_path) dataset = import_python_file(self.derived_dataset_path) self.assertAlmostEqual(dataset.dis_videos[0]['groundtruth'], 1.42307692308, places=4)
def from_dataset_file(cls, dataset_filepath, content_ids=None): dataset = import_python_file(dataset_filepath) if content_ids is not None: dataset.dis_videos = filter(lambda dis_video: dis_video['content_id'] in content_ids, dataset.dis_videos) dataset_reader = RawDatasetReader(dataset) return cls(dataset_reader)
def test_read_dataset_fps_rebuf_indices(self): train_dataset_path = VmafConfig.test_resource_path( 'test_dataset_fps_rebufinds.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertTrue('fps' in train_assets[0].asset_dict.keys()) self.assertTrue('rebuf_indices' in train_assets[0].asset_dict.keys())
def test_mos_subjective_model_output_custom_resampling(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) subjective_model.run_modeling() subjective_model.to_aggregated_dataset_file( self.output_dataset_filepath, resampling_type='lanczos') self.assertTrue(os.path.exists(self.output_dataset_filepath)) dataset2 = import_python_file(self.output_dataset_filepath) self.assertFalse(hasattr(dataset2, 'quality_height')) self.assertFalse(hasattr(dataset2, 'quality_width')) self.assertEquals(dataset2.resampling_type, 'lanczos') dis_video = dataset2.dis_videos[0] self.assertTrue('groundtruth' in dis_video) self.assertTrue('os' not in dis_video) self.assertAlmostEquals(dis_video['groundtruth'], 4.884615384615385, places=4)
def test_mos_subjective_model_normalize_final(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling(normalize_final=True) scores = result['quality_scores'] self.assertAlmostEquals(scores[0], 1.1318646945818083, places=4) self.assertAlmostEquals(scores[10], -1.2400334499143002, places=4) self.assertAlmostEquals(np.mean(scores), 0.0, places=4)
def test_mos_subjective_model_transform_final(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling(transform_final={'p1': 10, 'p0': 1}) scores = result['quality_scores'] self.assertAlmostEquals(scores[0], 49.84615384615385, places=4) self.assertAlmostEquals(scores[10], 21.769230769230771, places=4) self.assertAlmostEquals(np.mean(scores), 36.44790652385589, places=4)
def test_read_dataset_duration_sec(self): train_dataset_path = VmafConfig.test_resource_path('example_dataset_crop_pad_duration_sec.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEqual(len(train_assets), 3) self.assertEqual(train_assets[0].asset_dict['duration_sec'], 5.0) self.assertEqual(train_assets[1].asset_dict['duration_sec'], 5.0) self.assertEqual(train_assets[2].asset_dict['duration_sec'], 5.0)
def test_read_dataset_crop_and_pad(self): train_dataset_path = VmafConfig.test_resource_path('example_dataset_crop_pad.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 3) self.assertEquals(str(train_assets[0]), "example_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81") self.assertEquals(str(train_assets[1]), "example_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_padiw+100:ih+100:50:50") self.assertEquals(str(train_assets[2]), "example_0_3_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81_padiw+288:ih+162:144:81")
def test_mos_subjective_model(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling() scores = result['quality_scores'] self.assertAlmostEquals(scores[0], 4.884615384615385, places=4) self.assertAlmostEquals(scores[10], 2.0769230769230771, places=4) self.assertAlmostEquals(np.mean(scores), 3.544790652385589, places=4)
def test_read_dataset_crop_and_pad(self): train_dataset_path = VmafConfig.test_resource_path('example_dataset_crop_pad.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEqual(len(train_assets), 3) self.assertEqual(str(train_assets[0]), "example_0_1_src01_hrc00_576x324_576x324_crop288:162:144:81_vs_src01_hrc01_576x324_576x324_crop288:162:144:81_q_576x324") self.assertEqual(str(train_assets[1]), "example_0_2_src01_hrc00_576x324_576x324_padiw+100:ih+100:50:50_vs_src01_hrc01_576x324_576x324_padiw+100:ih+100:50:50_q_576x324") self.assertEqual(str(train_assets[2]), "example_0_3_src01_hrc00_576x324_576x324_crop288:162:144:81_padiw+288:ih+162:144:81_vs_src01_hrc01_576x324_576x324_crop288:162:144:81_padiw+288:ih+162:144:81_q_576x324")
def test_read_image_dataset_notyuv(self): dataset_path = VmafConfig.test_resource_path('test_image_dataset_notyuv.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEquals(len(assets), 4) self.assertTrue(assets[0].ref_width_height is None) self.assertTrue(assets[0].dis_width_height is None) self.assertEquals(assets[0].quality_width_height, (1920, 1080))
def test_generate_dataset_from_raw_mos(self): generate_dataset_from_raw( raw_dataset_filepath=self.raw_dataset_filepath, output_dataset_filepath=self.derived_dataset_path, subj_model_class=MosModel) dataset = import_python_file(self.derived_dataset_path) self.assertAlmostEqual(dataset.dis_videos[0]['groundtruth'], 1.3076923076923077, places=4)
def test_explain_train_test_model(self): model_class = SklearnRandomForestTrainTestModel train_dataset_path = VmafConfig.test_resource_path('test_image_dataset_diffdim.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) fextractor = MomentNorefFeatureExtractor( train_assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, ) fextractor.run(parallelize=True) self.features = fextractor.results xys = model_class.get_xys_from_results(self.features[:7]) model = model_class({'norm_type':'normalize', 'random_state':0}, None) model.train(xys) np.random.seed(0) xs = model_class.get_xs_from_results(self.features[7:]) explainer = LocalExplainer(neighbor_samples=1000) exps = explainer.explain(model, xs) self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4) self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4) self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4) self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4) self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4) self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4) self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4) self.assertEqual(exps['feature_names'], ['Moment_noref_feature_1st_score', 'Moment_noref_feature_2nd_score', 'Moment_noref_feature_var_score'] )
def test_observer_content_aware_subjective_model_subjreject(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) with self.assertRaises(AssertionError): subjective_model.run_modeling(subject_rejection=True)
def test_read_image_dataset_notyuv(self): dataset_path = VmafConfig.test_resource_path( 'test_image_dataset_notyuv.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEquals(len(assets), 4) self.assertTrue(assets[0].ref_width_height is None) self.assertTrue(assets[0].dis_width_height is None) self.assertEquals(assets[0].quality_width_height, (1920, 1080))
def test_test_on_dataset_split_test_indices_for_perf_ci(self): from vmaf.routine import run_test_on_dataset test_dataset = import_python_file(VmafConfig.test_resource_path('dataset_sample.py')) test_assets, results = run_test_on_dataset(test_dataset, VmafQualityRunner, None, None, None, parallelize=False, aggregate_method=None, split_test_indices_for_perf_ci=True, n_splits_test_indices=10) self.assertAlmostEqual(results[0]['VMAF_score'], 99.142659046424384, places=4)
def from_dataset_file(cls, dataset_filepath, content_ids=None): dataset = import_python_file(dataset_filepath) if content_ids is not None: dataset.dis_videos = filter( lambda dis_video: dis_video['content_id'] in content_ids, dataset.dis_videos) dataset_reader = RawDatasetReader(dataset) return cls(dataset_reader)
def run_vmaf_cv(train_dataset_filepath, test_dataset_filepath, param_filepath, output_model_filepath=None, **kwargs): result_store_dir = kwargs['result_store_dir'] if 'result_store_dir' in kwargs else VmafConfig.file_result_store_path() logger = get_stdout_logger() result_store = FileSystemResultStore(result_store_dir) train_dataset = import_python_file(train_dataset_filepath) test_dataset = import_python_file(test_dataset_filepath) if test_dataset_filepath is not None else None param = import_python_file(param_filepath) # === plot scatter === nrows = 1 ncols = 2 fig, axs = plt.subplots(figsize=(5*ncols, 5*nrows), nrows=nrows, ncols=ncols) train_test_vmaf_on_dataset(train_dataset, test_dataset, param, param, axs[0], axs[1], result_store, parallelize=True, logger=None, output_model_filepath=output_model_filepath, **kwargs) if 'xlim' in kwargs: axs[0].set_xlim(kwargs['xlim']) axs[1].set_xlim(kwargs['xlim']) if 'ylim' in kwargs: axs[0].set_ylim(kwargs['ylim']) axs[1].set_ylim(kwargs['ylim']) bbox = {'facecolor':'white', 'alpha':1, 'pad':20} axs[0].annotate('Training Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) axs[1].annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) plt.tight_layout() # === clean up === close_logger(logger)
def test_train_test_on_dataset_with_dis1st_thr_with_feature_optional_dict_good(self): from vmaf.routine import train_test_vmaf_on_dataset train_dataset = import_python_file( VmafConfig.test_resource_path('dataset_sample.py')) model_param = import_python_file( VmafConfig.test_resource_path('model_param_sample.py')) feature_param = import_python_file( VmafConfig.test_resource_path('feature_param_sample_with_optional_dict_good.py')) train_fassembler, train_assets, train_stats, test_fassembler, test_assets, test_stats, _ = train_test_vmaf_on_dataset( train_dataset=train_dataset, test_dataset=train_dataset, feature_param=feature_param, model_param=model_param, train_ax=None, test_ax=None, result_store=None, parallelize=True, logger=None, fifo_mode=True, output_model_filepath=self.output_model_filepath, ) self.train_fassembler = train_fassembler self.assertTrue(os.path.exists(self.output_model_filepath)) self.assertAlmostEqual(train_stats['ys_label_pred'][0], 90.753010402770798, places=3) self.assertAlmostEqual(test_stats['ys_label_pred'][0], 90.753010402770798, places=3) runner = VmafQualityRunner( train_assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={'model_filepath': self.output_model_filepath} ) runner.run(parallelize=True) results = runner.results self.assertAlmostEqual(results[0]['VMAF_score'], 89.55494473011981, places=4) self.assertAlmostEqual(results[1]['VMAF_score'], 61.01289549048653, places=4) self.assertAlmostEqual(results[2]['VMAF_score'], 90.75301241304798, places=4) self.assertAlmostEqual(results[3]['VMAF_score'], 89.27013895870179, places=4)
def setUp(self): dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py' dataset = import_python_file(dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } self.dataset_reader = CorruptSubjectRawDatasetReader( dataset, input_dict=info_dict)
def setUp(self): dataset_filepath = config.ROOT + '/python/test/resource/NFLX_dataset_public_raw.py' dataset = import_python_file(dataset_filepath) np.random.seed(0) info_dict = { 'corrupt_probability': 0.1, } self.dataset_reader = CorruptDataRawDatasetReader(dataset, input_dict=info_dict)
def test_mos_subjective_model(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling() scores = result['quality_scores'] self.assertAlmostEquals(scores[0], 4.884615384615385, places=4) self.assertAlmostEquals(scores[10], 2.0769230769230771, places=4) self.assertAlmostEquals(np.mean(scores), 3.544790652385589, places=4) scores_std = result['quality_scores_std'] self.assertAlmostEquals(np.mean(scores_std), 0.12986637295658307, places=4)
def run_vmaf_cv(train_dataset_filepath, test_dataset_filepath, param_filepath, output_model_filepath=None, **kwargs): logger = get_stdout_logger() result_store = FileSystemResultStore() train_dataset = import_python_file(train_dataset_filepath) test_dataset = import_python_file(test_dataset_filepath) if test_dataset_filepath is not None else None param = import_python_file(param_filepath) # === plot scatter === nrows = 1 ncols = 2 fig, axs = plt.subplots(figsize=(5*ncols, 5*nrows), nrows=nrows, ncols=ncols) train_test_vmaf_on_dataset(train_dataset, test_dataset, param, param, axs[0], axs[1], result_store, parallelize=True, logger=None, output_model_filepath=output_model_filepath, **kwargs) if 'xlim' in kwargs: axs[0].set_xlim(kwargs['xlim']) axs[1].set_xlim(kwargs['xlim']) if 'ylim' in kwargs: axs[0].set_ylim(kwargs['ylim']) axs[1].set_ylim(kwargs['ylim']) bbox = {'facecolor':'white', 'alpha':1, 'pad':20} axs[0].annotate('Training Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) axs[1].annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) plt.tight_layout() # === clean up === close_logger(logger)
def _import_dataset_and_filter(cls, dataset_filepath, content_ids, asset_ids): dataset = import_python_file(dataset_filepath) if content_ids is not None: dataset.dis_videos = filter( lambda dis_video: dis_video['content_id'] in content_ids, dataset.dis_videos) if asset_ids is not None: dataset.dis_videos = filter( lambda dis_video: dis_video['asset_id'] in asset_ids, dataset.dis_videos) return dataset
def test_mos_subjective_model_output2(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) subjective_model.run_modeling() dataset2 = subjective_model.to_aggregated_dataset() dis_video = dataset2.dis_videos[0] self.assertTrue('groundtruth' in dis_video) self.assertTrue('os' not in dis_video) self.assertAlmostEquals(dis_video['groundtruth'], 4.884615384615385, places=4)
def test_mos_subjective_model_transform_final(self): dataset = import_python_file(self.dataset_filepath) dataset_reader = RawDatasetReader(dataset) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling(transform_final={ 'p1': 10, 'p0': 1 }) scores = result['quality_scores'] self.assertAlmostEquals(scores[0], 49.84615384615385, places=4) self.assertAlmostEquals(scores[10], 21.769230769230771, places=4) self.assertAlmostEquals(np.mean(scores), 36.44790652385589, places=4)
def setUp(self): dataset_filepath = VmafConfig.test_resource_path( 'NFLX_dataset_public_raw.py') dataset = import_python_file(dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } self.dataset_reader = SelectSubjectRawDatasetReader( dataset, input_dict=info_dict)
def setUp(self): dataset_filepath = VmafConfig.test_resource_path( 'NFLX_dataset_public_raw.py') dataset = import_python_file(dataset_filepath) np.random.seed(0) info_dict = { 'corrupt_probability': 0.1, } self.dataset_reader = CorruptDataRawDatasetReader(dataset, input_dict=info_dict)
def test_observer_aware_subjective_model_corruptdata(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModelReduced(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.mean(result['quality_scores']), 3.5573073781669944, places=4) # 3.5482845335713469 self.assertAlmostEquals(np.var(result['quality_scores']), 1.3559834438740614, places=4) # 1.4355485462027884
def test_zscoresubjrejdmos_subjective_model_corruptdata_subjreject(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = ZscoringSubjrejDmosModel(dataset_reader) result = subjective_model.run_modeling() scores = result['quality_scores'] self.assertAlmostEquals(np.mean(scores), 0, places=4) self.assertAlmostEquals(np.var(scores), 0.66405245792414114, places=4) # 1.4012220200639218
def test_import_python_file(self): ds = import_python_file( VmafConfig.test_resource_path('example_raw_dataset2.py')) self.assertEqual(ds.dataset_name, 'example') self.assertEqual(ds.yuv_fmt, 'yuv420p') self.assertEqual(ds.ref_videos[0]['content_id'], 0) self.assertEqual(ds.ref_videos[0]['path'], '/dir/yuv/checkerboard_1920_1080_10_3_0_0.yuv') self.assertEqual(ds.dis_videos[3]['content_id'], 1) self.assertEqual(ds.dis_videos[3]['asset_id'], 3) self.assertEqual(ds.dis_videos[3]['os'], [70, 75, 80, 85, 90]) self.assertEqual(ds.dis_videos[3]['path'], '/dir/yuv/flat_1920_1080_10.yuv')
def run_vmaf_kfold_cv(dataset_filepath, contentid_groups, param_filepath, aggregate_method, ): logger = get_stdout_logger() result_store = FileSystemResultStore() dataset = import_python_file(dataset_filepath) param = import_python_file(param_filepath) fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1) cv_on_dataset(dataset, param, param, ax, result_store, contentid_groups, logger, aggregate_method) ax.set_xlim([0, 120]) ax.set_ylim([0, 120]) plt.tight_layout() # === clean up === close_logger(logger)
def test_read_dataset_refdif_start_end_frame(self): dataset_path = VmafConfig.test_resource_path('test_read_dataset_dataset3.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEqual(len(assets), 3) self.assertEqual(assets[0].ref_start_end_frame, (250, 250)) self.assertEqual(assets[0].dis_start_end_frame, (250, 250)) self.assertEqual(assets[1].ref_start_end_frame, (250, 250)) self.assertEqual(assets[1].dis_start_end_frame, (200, 210)) self.assertEqual(assets[2].ref_start_end_frame, (250, 250)) self.assertEqual(assets[2].dis_start_end_frame, (250, 251))
def test_subjrejdmos_subjective_model_corruptdata_subjreject(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = SubjrejDmosModel(dataset_reader) result = subjective_model.run_modeling() scores = result['quality_scores'] self.assertAlmostEquals(np.mean(scores), 4.0246673158065542, places=4) self.assertAlmostEquals(np.var(scores), 1.0932580358187849, places=4) # 1.4012220200639218
def test_zscore_mos_subjective_model_corruptdata_subjreject(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling(zscore_mode=True, subject_rejection=True) scores = result['quality_scores'] self.assertAlmostEquals(np.mean(scores), 0.0, places=4) self.assertAlmostEquals(np.var(scores), 0.66670826882879042, places=4)
def test_subjrejmos_subjective_model_corruptdata_subjreject(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = SubjrejMosModel(dataset_reader) result = subjective_model.run_modeling() scores = result['quality_scores'] self.assertAlmostEquals(np.mean(scores), 3.5611814345991566, places=4) self.assertAlmostEquals(np.var(scores), 1.1049505732699529, places=4) # 1.4012220200639218
def test_mos_subjective_model_corruptdata(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'selected_subjects': range(5), } dataset_reader = CorruptSubjectRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MosModel(dataset_reader) result = subjective_model.run_modeling() scores = result['quality_scores'] self.assertAlmostEquals(np.mean(scores), 3.5447906523855899, places=4) self.assertAlmostEquals(np.var(scores), 0.95893305294535369, places=4) # 1.4012220200639218
def test_read_dataset_basic(self): dataset_path = VmafConfig.test_resource_path('test_dataset.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEqual(len(assets), 4) self.assertTrue('groundtruth' in assets[0].asset_dict.keys()) self.assertTrue('os' not in assets[0].asset_dict.keys()) self.assertEqual(assets[0].quality_width_height, (1920, 1080)) self.assertEqual(assets[0].resampling_type, 'bicubic') self.assertEqual(assets[0].ref_yuv_type, 'yuv420p') self.assertEqual(assets[0].dis_yuv_type, 'yuv420p') self.assertEqual(assets[1].quality_width_height, (1920, 1080)) self.assertEqual(assets[1].resampling_type, 'bicubic')
def test_read_dataset_diffyuv(self): train_dataset_path = VmafConfig.test_resource_path('test_dataset_diffyuv.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 4) self.assertEquals(train_assets[0].ref_width_height, (1920, 1080)) self.assertEquals(train_assets[0].dis_width_height, (1920, 1080)) self.assertEquals(train_assets[0].quality_width_height, (1920, 1080)) self.assertEquals(train_assets[0].dis_yuv_type, 'yuv420p') self.assertEquals(train_assets[2].ref_width_height, (1280, 720)) self.assertEquals(train_assets[2].dis_width_height, (1280, 720)) self.assertEquals(train_assets[2].quality_width_height, (1280, 720)) self.assertEquals(train_assets[2].dis_yuv_type, 'yuv420p10le')
def test_read_dataset_qualitywh2(self): train_dataset_path = VmafConfig.test_resource_path('test_image_dataset_diffdim_qualitywh2.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertTrue('quality_width' in train_assets[0].asset_dict.keys()) self.assertTrue('quality_height' in train_assets[0].asset_dict.keys()) self.assertTrue('resampling_type' in train_assets[0].asset_dict.keys()) self.assertTrue('quality_width' not in train_assets[1].asset_dict.keys()) self.assertTrue('quality_height' not in train_assets[1].asset_dict.keys()) self.assertTrue('resampling_type' not in train_assets[1].asset_dict.keys()) self.assertEqual(train_assets[0].asset_dict['quality_width'], 200) self.assertEqual(train_assets[0].asset_dict['quality_height'], 100) self.assertEqual(train_assets[0].asset_dict['resampling_type'], 'bicubic')
def test_read_dataset(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 9) self.assertTrue('groundtruth' in train_assets[0].asset_dict.keys()) self.assertTrue('os' not in train_assets[0].asset_dict.keys()) self.assertTrue('width' in train_assets[0].asset_dict.keys()) self.assertTrue('height' in train_assets[0].asset_dict.keys()) self.assertTrue( 'quality_width' not in train_assets[0].asset_dict.keys()) self.assertTrue( 'quality_height' not in train_assets[0].asset_dict.keys())
def test_read_dataset_basic(self): dataset_path = VmafConfig.test_resource_path('test_dataset.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEquals(len(assets), 4) self.assertTrue('groundtruth' in assets[0].asset_dict.keys()) self.assertTrue('os' not in assets[0].asset_dict.keys()) self.assertEqual(assets[0].quality_width_height, (1920, 1080)) self.assertEqual(assets[0].resampling_type, 'bilinear') self.assertEqual(assets[0].ref_yuv_type, 'yuv420p') self.assertEqual(assets[0].dis_yuv_type, 'yuv420p') self.assertEqual(assets[1].quality_width_height, (1920, 1080)) self.assertEqual(assets[1].resampling_type, 'bilinear')
def explain_model_on_dataset( model, test_assets_selected_indexs, test_dataset_filepath, result_store_dir=VmafConfig.file_result_store_path()): def print_assets(test_assets): print('\n'.join( map( lambda tasset: "Asset {i}: {name}".format( i=tasset[0], name=get_file_name_without_extension(tasset[1].dis_path)), enumerate(test_assets)))) test_dataset = import_python_file(test_dataset_filepath) test_assets = read_dataset(test_dataset) print_assets(test_assets) print("Assets selected for local explanation: {}".format( test_assets_selected_indexs)) result_store = FileSystemResultStore(result_store_dir) test_assets = [test_assets[i] for i in test_assets_selected_indexs] test_fassembler = FeatureAssembler( feature_dict=model.model_dict['feature_dict'], feature_option_dict=None, assets=test_assets, logger=None, fifo_mode=True, delete_workdir=True, result_store=result_store, optional_dict=None, optional_dict2=None, parallelize=True, ) test_fassembler.run() test_feature_results = test_fassembler.results test_xs = model.get_xs_from_results(test_feature_results) test_ys = model.get_ys_from_results(test_feature_results) test_ys_pred = model.predict(test_xs)['ys_label_pred'] explainer = LocalExplainer(neighbor_samples=1000) test_exps = explainer.explain(model, test_xs) explainer.print_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred) explainer.plot_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred) DisplayConfig.show()
def test_read_dataset(self): train_dataset_path = VmafConfig.test_resource_path('test_image_dataset_diffdim.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 9) self.assertTrue('groundtruth' in train_assets[0].asset_dict.keys()) self.assertTrue('os' not in train_assets[0].asset_dict.keys()) self.assertFalse('width' in train_assets[0].asset_dict.keys()) self.assertTrue('ref_width' in train_assets[0].asset_dict.keys()) self.assertTrue('dis_width' in train_assets[0].asset_dict.keys()) self.assertFalse('height' in train_assets[0].asset_dict.keys()) self.assertTrue('ref_height' in train_assets[0].asset_dict.keys()) self.assertTrue('dis_height' in train_assets[0].asset_dict.keys()) self.assertTrue('quality_width' not in train_assets[0].asset_dict.keys()) self.assertTrue('quality_height' not in train_assets[0].asset_dict.keys())
def test_test_on_dataset_raw(self): test_dataset = import_python_file( VmafConfig.test_resource_path('raw_dataset_sample.py')) test_assets, results = run_test_on_dataset(test_dataset, VmafQualityRunner, None, None, None, parallelize=True, aggregate_method=None) self.assertAlmostEqual(results[0]['VMAF_score'], 99.142659046424384, places=4) self.assertAlmostEqual(results[1]['VMAF_score'], 35.066157497128764, places=4) self.assertAlmostEqual(results[2]['VMAF_score'], 97.428042675471147, places=4) self.assertAlmostEqual(results[3]['VMAF_score'], 97.427927701008869, places=4) self.assertAlmostEqual(test_assets[0].groundtruth, 100, places=4) self.assertAlmostEqual(test_assets[1].groundtruth, 50, places=4) self.assertAlmostEqual(test_assets[2].groundtruth, 100, places=4) self.assertAlmostEqual(test_assets[3].groundtruth, 90, places=4)
def setUp(self): train_dataset_path = VmafConfig.test_resource_path('test_image_dataset_diffdim.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) runner = MomentNorefFeatureExtractor( train_assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, ) runner.run(parallelize=True) self.features = runner.results
def test_observer_content_aware_subjective_model_missingdata(self): dataset = import_python_file(self.dataset_filepath) np.random.seed(0) info_dict = { 'missing_probability': 0.1, } dataset_reader = MissingDataRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_ambiguity']), 3.9104244772977128, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.0037713583509767193, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), -0.21903272050455846, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.084353684687185043, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 9.8168943054654481, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.028159236075789944, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.05548186797336, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4339487982797514, places=4) np.random.seed(0) info_dict = { 'missing_probability': 0.5, } dataset_reader = MissingDataRawDatasetReader(dataset, input_dict=info_dict) subjective_model = MaximumLikelihoodEstimationModel(dataset_reader) result = subjective_model.run_modeling() self.assertAlmostEquals(np.sum(result['content_ambiguity']), 2.63184284168883, places=4) self.assertAlmostEquals(np.var(result['content_ambiguity']), 0.019164097909450246, places=4) self.assertAlmostEquals(np.sum(result['observer_bias']), 0.2263148440748638, places=4) self.assertAlmostEquals(np.var(result['observer_bias']), 0.070613033112114504, places=4) self.assertAlmostEquals(np.sum(result['observer_inconsistency']), 12.317917502439435, places=4) self.assertAlmostEquals(np.var(result['observer_inconsistency']), 0.029455722248727296, places=4) self.assertAlmostEquals(np.sum(result['quality_scores']), 280.29962156788139, places=4) self.assertAlmostEquals(np.var(result['quality_scores']), 1.4717366222424826, places=4)
def setUp(self): train_dataset_path = VmafConfig.test_resource_path("test_image_dataset_diffdim.py") train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.h5py_filepath = VmafConfig.workdir_path('test.hdf5') self.h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath) optional_dict2 = {'h5py_file': self.h5py_file} fextractor = DisYUVRawVideoExtractor( train_assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=optional_dict2, ) fextractor.run(parallelize=False) # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor self.features = fextractor.results