class ResultStoreTest(unittest.TestCase): def setUp(self): ref_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_0_0.yuv" dis_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_1_0.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':1920, 'height':1080}) self.runner = VmafLegacyQualityRunner( [asset], None, fifo_mode=True, delete_workdir=True, result_store=None, ) self.runner.run() self.result = self.runner.results[0] def tearDown(self): if hasattr(self, 'result') and hasattr(self, 'result_store'): self.result_store.delete(self.result.asset, self.result.executor_id) pass def test_file_system_result_store_save_load(self): print 'test on file system result store save and load...' self.result_store = FileSystemResultStore(logger=None) asset = self.result.asset executor_id = self.result.executor_id self.result_store.save(self.result) loaded_result = self.result_store.load(asset, executor_id) self.assertEquals(self.result, loaded_result)
def test_file_system_result_store_save_load(self): print 'test on file system result store save and load...' self.result_store = FileSystemResultStore(logger=None) asset = self.result.asset executor_id = self.result.executor_id self.result_store.save(self.result) loaded_result = self.result_store.load(asset, executor_id) self.assertEquals(self.result, loaded_result)
def test_run_vamf_legacy_runner_with_result_store(self): print 'test on running VMAF (legacy) runner with result store...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) result_store = FileSystemResultStore(logger=None) self.runner = VmafLegacyQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=result_store ) print ' running for the first time with fresh calculation...' self.runner.run() result0, result1 = self.runner.results # NOTE: since stored results are actually VMAF_feature's not VMAF's, # the two paths below shouldn't exist self.assertFalse(os.path.exists(result_store._get_result_file_path(result0))) self.assertFalse(os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAF_legacy_score'], 60.27316952679754) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[1]['VMAF_legacy_score'], 95.65756240092573) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def main(): if len(sys.argv) < 3: print_usage() return 2 try: quality_type = sys.argv[1] dataset_filepath = sys.argv[2] except ValueError: print_usage() return 2 try: dataset = import_python_file(dataset_filepath) except Exception as e: print "Error: " + str(e) return 1 try: runner_class = QualityRunner.find_subclass(quality_type) except: print_usage() return 2 result_store = FileSystemResultStore() run_remove_results_for_dataset(result_store, dataset, runner_class) return 0
def run_vmaf_cv(train_dataset_filepath, test_dataset_filepath, param_filepath, output_model_filepath=None, **kwargs): logger = get_stdout_logger() result_store = FileSystemResultStore() train_dataset = import_python_file(train_dataset_filepath) test_dataset = import_python_file( test_dataset_filepath) if test_dataset_filepath is not None else None param = import_python_file(param_filepath) # === plot scatter === nrows = 1 ncols = 2 fig, axs = plt.subplots(figsize=(5 * ncols, 5 * nrows), nrows=nrows, ncols=ncols) train_test_vmaf_on_dataset(train_dataset, test_dataset, param, param, axs[0], axs[1], result_store, parallelize=True, logger=None, output_model_filepath=output_model_filepath, **kwargs) if 'xlim' in kwargs: axs[0].set_xlim(kwargs['xlim']) axs[1].set_xlim(kwargs['xlim']) if 'ylim' in kwargs: axs[0].set_ylim(kwargs['ylim']) axs[1].set_ylim(kwargs['ylim']) bbox = {'facecolor': 'white', 'alpha': 1, 'pad': 20} axs[0].annotate('Training Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) axs[1].annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) plt.tight_layout() # === clean up === close_logger(logger)
def setUp(self): ref_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_0_0.yuv" dis_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_1_0.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':1920, 'height':1080}) self.runner = VmafLegacyQualityRunner( [asset], None, fifo_mode=True, delete_workdir=True, result_store=FileSystemResultStore(), ) self.runner.run() self.result = self.runner.results[0]
def explain_model_on_dataset(model, test_assets_selected_indexs, test_dataset_filepath): def print_assets(test_assets): print '\n'.join( map( lambda (i, asset): "Asset {i}: {name}".format( i=i, name=get_file_name_without_extension(asset.dis_path)), enumerate(test_assets))) test_dataset = import_python_file(test_dataset_filepath) test_assets = read_dataset(test_dataset) print_assets(test_assets) print "Assets selected for local explanation: {}".format( test_assets_selected_indexs) result_store = FileSystemResultStore() test_assets = [test_assets[i] for i in test_assets_selected_indexs] test_fassembler = FeatureAssembler( feature_dict=model.model_dict['feature_dict'], feature_option_dict=None, assets=test_assets, logger=None, fifo_mode=True, delete_workdir=True, result_store=result_store, optional_dict=None, optional_dict2=None, parallelize=True, ) test_fassembler.run() test_feature_results = test_fassembler.results test_xs = model.get_xs_from_results(test_feature_results) test_ys = model.get_ys_from_results(test_feature_results) test_ys_pred = model.predict(test_xs) explainer = LocalExplainer(neighbor_samples=1000) test_exps = explainer.explain(model, test_xs) explainer.print_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred) explainer.plot_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred) plt.show()
def run_vmaf_kfold_cv(dataset_filepath, contentid_groups, param_filepath, aggregate_method, ): logger = get_stdout_logger() result_store = FileSystemResultStore() dataset = import_python_file(dataset_filepath) param = import_python_file(param_filepath) fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1) cv_on_dataset(dataset, param, param, ax, result_store, contentid_groups, logger, aggregate_method) ax.set_xlim([0, 120]) ax.set_ylim([0, 120]) plt.tight_layout() # === clean up === close_logger(logger)
def test_run_parallel_vamf_fextractor_with_result_store(self): print "test on running VMAF feature extractor with result store " "in parallel..." ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset( dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={"width": 576, "height": 324}, ) asset_original = Asset( dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={"width": 576, "height": 324}, ) result_store = FileSystemResultStore(logger=None) print " running for the first time with fresh calculation..." self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print " running for the second time with stored results..." _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]["VMAF_feature_vif_score"], 0.4460930625, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_motion_score"], 4.04982535417, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm2_score"], 0.925421075027, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_ansnr_score"], 23.5095715208, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_num_score"], 712650.023478, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_den_score"], 1597314.95249, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_num_score"], 6899.24648475, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_den_score"], 7535.29963308, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_anpsnr_score"], 34.164776875, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale0_score"], 0.363420489439, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale1_score"], 0.766647542135, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale2_score"], 0.862854666902, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale3_score"], 0.915971778036, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale0_score"], 0.86711114907261122, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale1_score"], 0.87407085185224942, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale2_score"], 0.92448300466700284, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale3_score"], 0.96367203202196239, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif2_score"], 0.72722361912801026, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm3_score"], 0.90733425940345658, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_motion_score"], 4.04982535417, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_ansnr_score"], 31.2714392708, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_num_score"], 1597314.86733, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_den_score"], 1597314.95249, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_num_score"], 7535.29963308, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_den_score"], 7535.29963308, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_anpsnr_score"], 41.9266444375, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm3_score"], 1.0, places=4)
def test_run_parallel_vamf_fextractor_with_result_store(self): print 'test on running VMAF feature extractor with result store ' \ 'in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) result_store = FileSystemResultStore(logger=None) print ' running for the first time with fresh calculation...' self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.925421075027, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.24648475, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.29963308, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.86711114907261122, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.87407085185224942, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.92448300466700284, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96367203202196239, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.90733425940345658, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.29963308, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.29963308, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
def setUp(self): self.result_store = FileSystemResultStore()
def test_run_parallel_vamf_fextractor_with_result_store(self): print 'test on running VMAF feature extractor with result store ' \ 'in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) result_store = FileSystemResultStore(logger=None) print ' running for the first time with fresh calculation...' self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499, places=4)
def test_run_vamf_legacy_runner_with_result_store(self): print 'test on running VMAF (legacy) runner with result store...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) result_store = FileSystemResultStore(logger=None) self.runner = VmafLegacyQualityRunner([asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=result_store) print ' running for the first time with fresh calculation...' self.runner.run() result0, result1 = self.runner.results # NOTE: since stored results are actually VMAF_feature's not VMAF's, # the two paths below shouldn't exist self.assertFalse( os.path.exists(result_store._get_result_file_path(result0))) self.assertFalse( os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAF_legacy_score'], 60.27316952679754, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329, places=4) self.assertAlmostEqual(results[1]['VMAF_legacy_score'], 95.65756240092573, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322, places=4)
feature_param = import_python_file(feature_param_filepath) model_param = import_python_file(model_param_filepath) except Exception as e: print "Error: " + str(e) exit(1) if do_parallelize == 'yes': parallelize = True elif do_parallelize == 'no': parallelize = False else: print_usage() exit(2) if cache_result == 'yes': result_store = FileSystemResultStore() elif cache_result == 'no': result_store = None else: print_usage() exit(2) try: import matplotlib.pyplot as plt fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1) train_test_on_dataset(train_dataset=train_dataset, test_dataset=None, feature_param=feature_param, model_param=model_param, train_ax=ax, test_ax=None, result_store=result_store, parallelize=parallelize,
def main(): if len(sys.argv) < 3: print_usage() return 2 try: quality_type = sys.argv[1] test_dataset_filepath = sys.argv[2] except ValueError: print_usage() return 2 vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv), '--vmaf-model') cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--cache-result') parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv), '--parallelize') print_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--print-result') suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv), '--suppress-plot') pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool') if not (pool_method is None or pool_method in POOL_METHODS): print '--pool can only have option among {}'.format(', '.join(POOL_METHODS)) return 2 subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model') try: if subj_model is not None: subj_model_class = SubjectiveModel.find_subclass(subj_model) else: subj_model_class = None except Exception as e: print "Error: " + str(e) return 1 if vmaf_model_path is not None and quality_type != VmafQualityRunner.TYPE: print "Input error: only quality_type of VMAF accepts --vmaf-model." print_usage() return 2 try: test_dataset = import_python_file(test_dataset_filepath) except Exception as e: print "Error: " + str(e) return 1 try: runner_class = QualityRunner.find_subclass(quality_type) except Exception as e: print "Error: " + str(e) return 1 if cache_result: result_store = FileSystemResultStore() else: result_store = None # pooling if pool_method == 'harmonic_mean': aggregate_method = ListStats.harmonic_mean elif pool_method == 'min': aggregate_method = np.min elif pool_method == 'median': aggregate_method = np.median elif pool_method == 'perc5': aggregate_method = ListStats.perc5 elif pool_method == 'perc10': aggregate_method = ListStats.perc10 elif pool_method == 'perc20': aggregate_method = ListStats.perc20 else: # None or 'mean' aggregate_method = np.mean try: if suppress_plot: raise AssertionError import matplotlib.pyplot as plt fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1) assets, results = test_on_dataset(test_dataset, runner_class, ax, result_store, vmaf_model_path, parallelize=parallelize, aggregate_method=aggregate_method, subj_model_class=subj_model_class, ) bbox = {'facecolor':'white', 'alpha':0.5, 'pad':20} ax.annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) # ax.set_xlim([-10, 110]) # ax.set_ylim([-10, 110]) plt.tight_layout() plt.show() except ImportError: print_matplotlib_warning() assets, results = test_on_dataset(test_dataset, runner_class, None, result_store, vmaf_model_path, parallelize=parallelize, aggregate_method=aggregate_method, subj_model_class=subj_model_class, ) except AssertionError: assets, results = test_on_dataset(test_dataset, runner_class, None, result_store, vmaf_model_path, parallelize=parallelize, aggregate_method=aggregate_method, subj_model_class=subj_model_class, ) if print_result: for result in results: print result print '' return 0
def test_run_parallel_vamf_fextractor_with_result_store(self): print 'test on running VMAF feature extractor with result store ' \ 'in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) result_store = FileSystemResultStore(logger=None) print ' running for the first time with fresh calculation...' self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue( os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue( os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499)
def main(): if len(sys.argv) < 5: print_usage() return 2 try: train_dataset_filepath = sys.argv[1] feature_param_filepath = sys.argv[2] model_param_filepath = sys.argv[3] output_model_filepath = sys.argv[4] except ValueError: print_usage() return 2 try: train_dataset = import_python_file(train_dataset_filepath) feature_param = import_python_file(feature_param_filepath) model_param = import_python_file(model_param_filepath) except Exception as e: print "Error: " + str(e) return 1 cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--cache-result') parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv), '--parallelize') pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool') if not (pool_method is None or pool_method in POOL_METHODS): print '--pool can only have option among {}'.format(', '.join(POOL_METHODS)) return 2 if cache_result: result_store = FileSystemResultStore() else: result_store = None # pooling if pool_method == 'harmonic_mean': aggregate_method = ListStats.harmonic_mean elif pool_method == 'min': aggregate_method = np.min elif pool_method == 'median': aggregate_method = np.median elif pool_method == 'perc5': aggregate_method = ListStats.perc5 elif pool_method == 'perc10': aggregate_method = ListStats.perc10 elif pool_method == 'perc20': aggregate_method = ListStats.perc20 else: # None or 'mean' aggregate_method = np.mean logger = None try: import matplotlib.pyplot as plt fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1) train_test_vmaf_on_dataset(train_dataset=train_dataset, test_dataset=None, feature_param=feature_param, model_param=model_param, train_ax=ax, test_ax=None, result_store=result_store, parallelize=parallelize, logger=logger, output_model_filepath=output_model_filepath, aggregate_method=aggregate_method ) bbox = {'facecolor':'white', 'alpha':0.5, 'pad':20} ax.annotate('Training Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox) # ax.set_xlim([-10, 110]) # ax.set_ylim([-10, 110]) plt.tight_layout() plt.show() except ImportError: print_matplotlib_warning() train_test_vmaf_on_dataset(train_dataset=train_dataset, test_dataset=None, feature_param=feature_param, model_param=model_param, train_ax=None, test_ax=None, result_store=result_store, parallelize=parallelize, logger=logger, output_model_filepath=output_model_filepath, aggregate_method=aggregate_method ) return 0