def test_run_vmaf_fextractor_with_resampling(self): ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324, 'quality_width':160, 'quality_height':90}) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324, 'quality_width':160, 'quality_height':90}) self.fextractor = VmafFeatureExtractor( [asset, asset_original], None, fifo_mode=False) self.fextractor.run(parallelize=False) results = self.fextractor.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.782546520833, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],1.3216766875, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.98229347916666665, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 28.0085990417, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 1.3216766875, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.811350125, places=4)
def test_run_strred_fextractor(self): print 'test on running STRRED feature extractor...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractor = StrredFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=True) results = self.fextractor.results self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0114681041666671, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.3039486249999994, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 21.995608318659482, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_checkerboard_identical(self): print 'test on checkerboard pattern identical...' LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_checkerboardIdentical' ref_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") dis_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") yuv_fmt = "yuv420p" yuv_width = 1920 yuv_height = 1080 cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv, dis=dis_yuv, w=yuv_width, h=yuv_height, log=LOCAL_LOG_FILENAME) run_process(cmd, shell=True) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 1.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], 21.1138813333, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 12.554836666666667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 1.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 2773.891225, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 2773.891225, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 33021350.5, places=-3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 33021387.0625, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 29.8567246667, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0], 25757432.0, places=-3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 25757473.3333, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 259774.958333, places=1) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 259774.9375, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 277.120382, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 277.120382, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 924.193766, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 924.193766, places=3)
def test_run_asset_extractor(self): print 'test on running asset extractor...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324, 'quality_width':160, 'quality_height':90}) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324, 'quality_width':160, 'quality_height':90}) self.fextractor = AssetExtractor( [asset, asset_original], None, fifo_mode=True) self.fextractor.run() results = self.fextractor.results self.assertEqual(str(results[0]['asset']), 'test_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_160x90') self.assertEqual(str(results[1]['asset']), 'test_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc00_576x324_576x324_q_160x90') self.fextractor.run(parallelize=True) results = self.fextractor.results self.assertEqual(str(results[0]['asset']), 'test_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_160x90') self.assertEqual(str(results[1]['asset']), 'test_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc00_576x324_576x324_q_160x90')
def test_checkerboard_shifted_by_1(self): print 'test on checkerboard pattern shifted by 1...' LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_checkerboard_shifted_by_1' ref_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") dis_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv") yuv_fmt = "yuv420p" yuv_width = 1920 yuv_height = 1080 cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv, dis=dis_yuv, w=yuv_width, h=yuv_height, log=LOCAL_LOG_FILENAME) run_process(cmd, shell=True) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 0.7853383333333334, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], 7.92623066667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 12.5548366667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 0.156834666667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 2178.5352886666665, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 2773.891225, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 5178894.51562, places=-1) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 33021387.0625, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 16.669074, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0], 2908829.0, places=-1) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 25757473.3333, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 128957.796875, places=-2) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 259774.9375, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 201.15329999999997, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 277.120382, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 765.1562903333333, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 924.193766, places=3)
def test_run_psnr_fextractor(self): print 'test on running PSNR feature extractor...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractor = PsnrFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['PSNR_feature_psnr_score'], 30.755063979166664, places=4) self.assertAlmostEqual(results[1]['PSNR_feature_psnr_score'], 60.0, places=4)
def test_explain_vmaf_results(self): print 'test on running VMAF runner with local explainer...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.runner = VmafQualityRunnerWithLocalExplainer( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict2={'explainer': LocalExplainer(neighbor_samples=100)} ) np.random.seed(0) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAF_score'], 76.699271272486044, places=4) self.assertAlmostEqual(results[1]['VMAF_score'], 99.946416604585025, places=4) expected_feature_names = ['VMAF_feature_adm2_score', 'VMAF_feature_motion2_score', 'VMAF_feature_vif_scale0_score', 'VMAF_feature_vif_scale1_score', 'VMAF_feature_vif_scale2_score', 'VMAF_feature_vif_scale3_score'] weights = np.mean(results[0]['VMAF_scores_exps']['feature_weights'], axis=0) self.assertAlmostEqual(weights[0], 0.66021689480916868, places=4) self.assertAlmostEqual(weights[1], 0.14691682562211777, places=4) self.assertAlmostEqual(weights[2], -0.023682744847036086, places=4) self.assertAlmostEqual(weights[3], -0.029779341850172818, places=4) self.assertAlmostEqual(weights[4], 0.19149485210137338, places=4) self.assertAlmostEqual(weights[5], 0.31890978778344126, places=4) self.assertEqual(results[0]['VMAF_scores_exps']['feature_names'], expected_feature_names) weights = np.mean(results[1]['VMAF_scores_exps']['feature_weights'], axis=0) self.assertAlmostEqual(weights[0], 0.69597961598838509, places=4) self.assertAlmostEqual(weights[1], 0.18256016705513464, places=4) self.assertAlmostEqual(weights[2], 0.0090048099912423147, places=4) self.assertAlmostEqual(weights[3], 0.028671810808880094, places=4) self.assertAlmostEqual(weights[4], 0.21935602577417926, places=4) self.assertAlmostEqual(weights[5], 0.34190431429767715, places=4) self.assertEqual(results[1]['VMAF_scores_exps']['feature_names'], expected_feature_names)
def test_flat_value10(self): print 'test on flat pattern of value 10...' LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_flat_value10' ref_yuv = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv") dis_yuv = VmafConfig.test_resource_path("yuv", "flat_1920_1080_10.yuv") yuv_fmt = "yuv420p" yuv_width = 1920 yuv_height = 1080 cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv, dis=dis_yuv, w=yuv_width, h=yuv_height, log=LOCAL_LOG_FILENAME) run_process(cmd, shell=True) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 1.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], 21.899511, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 0.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 1.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 149.780313, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 149.780392, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 2753999.99219, places=1) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 2754000.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 29.045954, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0],2073600.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 2073600.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 32400.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 32400.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 65.573967, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 65.573967, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 16.667711, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 16.667747, places=3)
def test_checkerboard_opposite(self): print 'test on checkerboard pattern opposite...' LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_checkerboard_opposite' ref_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") dis_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") yuv_fmt = "yuv420p" yuv_width = 1920 yuv_height = 1080 cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv, dis=dis_yuv, w=yuv_width, h=yuv_height, log=LOCAL_LOG_FILENAME) run_process(cmd, shell=True) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 0.053996333333333334, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], -5.758091333333334, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 12.554836666666667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 0.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 149.780313, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 2773.891225, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 6.66666666667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 33021387.0625, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 2.984752, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0], 6.66666666667, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 25757473.3333, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 0.0, places=4) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 259774.9375, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 65.573967, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 277.120382, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 16.667711, places=3) self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 924.193766, places=3)
def test_run_dis_yuv_raw_video_extractor_parallel(self): print 'test on running dis YUV raw video extractor...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath) self.fextractor = DisYUVRawVideoExtractor( [asset, asset_original], None, fifo_mode=False, optional_dict={'channels': 'yu'}, optional_dict2={'h5py_file': h5py_file} ) with self.assertRaises(AssertionError): self.fextractor.run(parallelize=True) DisYUVRawVideoExtractor.close_h5py_file(h5py_file)
def test_train_test_on_raw_dataset_with_dis1st_thr(self): train_dataset = import_python_file( VmafConfig.test_resource_path('raw_dataset_sample.py')) model_param = import_python_file( VmafConfig.test_resource_path('model_param_sample.py')) feature_param = import_python_file( VmafConfig.test_resource_path('feature_param_sample.py')) train_fassembler, train_assets, train_stats, test_fassembler, test_assets, test_stats, _ = train_test_vmaf_on_dataset( train_dataset=train_dataset, test_dataset=train_dataset, feature_param=feature_param, model_param=model_param, train_ax=None, test_ax=None, result_store=None, parallelize=True, logger=None, fifo_mode=True, output_model_filepath=self.output_model_filepath ) self.train_fassembler = train_fassembler self.assertTrue(os.path.exists(self.output_model_filepath)) self.assertAlmostEqual(train_stats['ys_label_pred'][0], 93.565459224020742, places=3) self.assertAlmostEqual(test_stats['ys_label_pred'][0], 93.565459224020742, places=3)
def test_run_parallel_moment_noref_fextractor(self): print 'test on running Moment noref feature extractor on NorefAssets in parallel...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = NorefAsset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = NorefAsset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), dis_path=ref_path, asset_dict={'width':576, 'height':324}) fextractor = MomentNorefFeatureExtractor( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, ) self.fextractors = [fextractor] fextractor.run(parallelize=True) results = fextractor.results self.assertAlmostEqual(results[0]['Moment_noref_feature_1st_score'], 61.332006624999984) self.assertAlmostEqual(results[0]['Moment_noref_feature_2nd_score'], 4798.659574041666) self.assertAlmostEqual(results[0]['Moment_noref_feature_var_score'], 1036.8371843488285) self.assertAlmostEqual(results[1]['Moment_noref_feature_1st_score'], 59.788567297525134) self.assertAlmostEqual(results[1]['Moment_noref_feature_2nd_score'], 4696.668388042271) self.assertAlmostEqual(results[1]['Moment_noref_feature_var_score'], 1121.519917231207)
def test_run_parallel_ssim_fextractor(self): print 'test on running SSIM feature extractor in parallel...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractor = SsimFeatureExtractor( [asset, asset_original, asset], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=True) results = self.fextractor.results self.assertAlmostEqual(results[0]['SSIM_feature_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_l_score'], 0.9981474583333334, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_c_score'], 0.96126793750000006, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_s_score'], 0.89773633333333336, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_l_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_c_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_s_score'], 1.0, places=4) self.assertAlmostEqual(results[2]['SSIM_feature_ssim_score'], 0.86322654166666657, places=4)
def test_run_psnr(self): exe = VmafConfig.root_path('run_psnr') line = 'yuv420p 576 324 {root}/python/test/resource/yuv/src01_hrc00_576x324.yuv ' \ '{root}/python/test/resource/yuv/src01_hrc01_576x324.yuv'.format(root=VmafConfig.root_path()) cmd = "{exe} {line} >/dev/null 2>&1".format(line=line, exe=exe) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0)
def test_run_ms_ssim_fextractor(self): print 'test on running MS-SSIM feature extractor...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractor = MsSsimFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9981474583333334, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.96126793750000006, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.89773633333333336, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.99899612500000001, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9857694375, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.941185875, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.99923564583333324, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.997034020833, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.977992145833, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.99929210416666658, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.999588104167, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.99387125, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.99940356249999995, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.999907625, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.998222583333, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_cleaning_cache_psnr(self): exe = VmafConfig.root_path('run_testing') cmd = "{exe} PSNR {dataset} --parallelize --cache-result --suppress-plot".format( exe=exe, dataset=self.dataset_filename) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0) exe = VmafConfig.root_path('python', 'script', 'run_cleaning_cache.py') cmd = "{exe} PSNR {dataset}".format( exe=exe, dataset=self.dataset_filename) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0)
def test_run_vmaf_in_batch(self): line = 'yuv420p 576 324 {root}/python/test/resource/yuv/src01_hrc00_576x324.yuv ' \ '{root}/python/test/resource/yuv/src01_hrc01_576x324.yuv'.format(root=VmafConfig.root_path()) cmd = 'echo "{line}" > {batch_filename}'.format( line=line, batch_filename=self.batch_filename) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0) exe = VmafConfig.root_path('run_vmaf_in_batch') cmd = "{exe} {input} --parallelize >/dev/null 2>&1".format( exe=exe, input=self.batch_filename) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0)
def test_bitrate(self): ref_path = VmafConfig.test_resource_path('yuv/', 'src01_hrc00_576x324.yuv') dis_path = VmafConfig.test_resource_path('yuv', 'src01_hrc01_576x324.yuv') asset = Asset(dataset="test", content_id=0, asset_id=0, ref_path=ref_path, dis_path=dis_path, asset_dict={'ref_start_frame':0, 'ref_end_frame':47, 'dis_start_frame':0, 'dis_end_frame':47, 'fps':23.976},) self.assertAlmostEquals(asset.ref_bitrate_kbps_for_entire_file, 53693.964287999996, places=4) self.assertAlmostEquals(asset.dis_bitrate_kbps_for_entire_file, 53693.964287999996, places=4)
def setUp(self): ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':1920, 'height':1080}) self.runner = VmafLegacyQualityRunner( [asset], None, fifo_mode=True, delete_workdir=True, result_store=None, ) self.runner.run() self.result = self.runner.results[0]
def test_run_testlib_runner(self): print 'test on running TESTLIB runner...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.runner = TestLibRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['TESTLIB_vif_scale0_score'],0.363420458333, places=4) self.assertAlmostEqual(results[0]['TESTLIB_vif_scale1_score'], 0.766647520833, places=4) self.assertAlmostEqual(results[0]['TESTLIB_vif_scale2_score'], 0.862854708333, places=4) self.assertAlmostEqual(results[0]['TESTLIB_vif_scale3_score'], 0.915971791667, places=4) self.assertAlmostEqual(results[0]['TESTLIB_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[0]['TESTLIB_adm2_score'], 0.93458777083333333, places=4) self.assertAlmostEqual(results[0]['TESTLIB_psnr_score'], 30.7550666667, places=4) self.assertAlmostEqual(results[0]['TESTLIB_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['TESTLIB_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[1]['TESTLIB_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['TESTLIB_vif_scale1_score'],0.999999958333, places=4) self.assertAlmostEqual(results[1]['TESTLIB_vif_scale2_score'],0.999999416667, places=4) self.assertAlmostEqual(results[1]['TESTLIB_vif_scale3_score'], 0.999999208333, places=4) self.assertAlmostEqual(results[1]['TESTLIB_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[1]['TESTLIB_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['TESTLIB_psnr_score'], 60.0, places=4) self.assertAlmostEqual(results[1]['TESTLIB_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['TESTLIB_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['TESTLIB_score'], 76.699271272486044, places=3) self.assertAlmostEqual(results[1]['TESTLIB_score'],99.946416604585025, places=4)
def run_vmaf_cv_from_raw(train_dataset_raw_filepath, test_dataset_raw_filepath, param_filepath, output_model_filepath, **kwargs): if 'train_quality_wh' in kwargs: train_quality_width, train_quality_height = kwargs['train_quality_wh'] else: train_quality_width = None train_quality_height = None if 'test_quality_wh' in kwargs: test_quality_width, test_quality_height = kwargs['test_quality_wh'] else: test_quality_width = None test_quality_height = None if 'train_transform_final' in kwargs: train_transform_final = kwargs['train_transform_final'] else: train_transform_final = None if 'test_transform_final' in kwargs: test_transform_final = kwargs['test_transform_final'] else: test_transform_final = None train_output_dataset_filepath = VmafConfig.workspace_path('dataset', 'train_dataset.py') generate_dataset_from_raw(raw_dataset_filepath=train_dataset_raw_filepath, output_dataset_filepath=train_output_dataset_filepath, quality_width=train_quality_width, quality_height=train_quality_height, transform_final=train_transform_final, **kwargs) test_output_dataset_filepath = VmafConfig.workspace_path('dataset', 'test_dataset.py') \ if test_dataset_raw_filepath is not None else None generate_dataset_from_raw(raw_dataset_filepath=test_dataset_raw_filepath, output_dataset_filepath=test_output_dataset_filepath, quality_width=test_quality_width, quality_height=test_quality_height, transform_final=test_transform_final, **kwargs) run_vmaf_cv( train_dataset_filepath=train_output_dataset_filepath, test_dataset_filepath=test_output_dataset_filepath, param_filepath=param_filepath, output_model_filepath=output_model_filepath, **kwargs )
def test_read_dataset_mixed(self): dataset_path = VmafConfig.test_resource_path('test_dataset_mixed.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEquals(len(assets), 4) self.assertEqual(assets[0].resampling_type, 'bilinear') self.assertEqual(assets[0].ref_yuv_type, 'yuv420p') self.assertEqual(assets[0].dis_yuv_type, 'yuv420p') self.assertEqual(assets[0].ref_width_height, (1920, 1080)) self.assertEqual(assets[0].dis_width_height, (1920, 1080)) self.assertEqual(assets[1].resampling_type, 'bilinear') self.assertEqual(assets[1].ref_yuv_type, 'yuv420p') self.assertEqual(assets[1].dis_yuv_type, 'notyuv') self.assertEqual(assets[1].ref_width_height, (1920, 1080)) self.assertEqual(assets[1].dis_width_height, None) self.assertEqual(assets[2].resampling_type, 'bilinear') self.assertEqual(assets[2].ref_yuv_type, 'yuv420p') self.assertEqual(assets[2].dis_yuv_type, 'yuv420p') self.assertEqual(assets[2].ref_width_height, (720, 480)) self.assertEqual(assets[2].dis_width_height, (720, 480)) self.assertEqual(assets[3].resampling_type, 'bilinear') self.assertEqual(assets[3].ref_yuv_type, 'yuv420p') self.assertEqual(assets[3].dis_yuv_type, 'notyuv') self.assertEqual(assets[3].ref_width_height, (720, 480)) self.assertEqual(assets[3].dis_width_height, None)
def test_run_strred_fextractor_blackframes(self): print 'test on running STRRED feature extractor on flat frames...' ref_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv") dis_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_10.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) from vmaf.core.result_store import FileSystemResultStore result_store = FileSystemResultStore(logger=None) self.fextractor = StrredFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=result_store ) print ' running for the first time with fresh calculation...' self.fextractor.run(parallelize=True) result0, result1 = self.fextractor.results import os self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' self.fextractor.run(parallelize=True) results = self.fextractor.results # ignore NaN for result in results: result.set_score_aggregate_method(ListStats.nonemean) self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 1220.5679849999999, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 50983.3097155, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 62228595.6081, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_feature_assembler_selected_atom_feature(self): print 'test on feature assembler with selected atom features...' ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fassembler = FeatureAssembler( feature_dict={'VMAF_feature':['vif', 'motion']}, feature_option_dict=None, assets=[asset, asset_original], logger=None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, parallelize=True, ) self.fassembler.run() results = self.fassembler.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4) with self.assertRaises(KeyError): results[0]['VMAF_feature_ansnr_scores'] with self.assertRaises(KeyError): results[0]['VMAF_feature_ansnr_score'] with self.assertRaises(KeyError): results[0]['VMAF_feature_adm_scores'] with self.assertRaises(KeyError): results[0]['VMAF_feature_adm_score']
def test_with(self): with YuvReader( filepath=VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv"), width=576, height=324, yuv_type='yuv420p' ) as yuv_reader: self.assertEquals(yuv_reader.file.__class__, file)
def test_kflk_metrics_performance(self): mat_filepath = VmafConfig.test_resource_path('data_Toyama.mat') mat_dict = scipy.io.loadmat(mat_filepath) results = KflkPerfMetric._metrics_performance(mat_dict['objScoDif'], mat_dict['signif']) self.assertAlmostEqual(np.mean(results['AUC_DS']), 0.69767003960902052, places=6) self.assertAlmostEqual(np.mean(results['AUC_BW']), 0.94454700301894534, places=6) self.assertAlmostEqual(np.mean(results['CC_0']), 0.88105386206276415, places=6) self.assertAlmostEqual(np.mean(results['THR']), 6.2392849606450556, places=6)
def test_read_dataset_crop_and_pad(self): train_dataset_path = VmafConfig.test_resource_path('example_dataset_crop_pad.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 3) self.assertEquals(str(train_assets[0]), "example_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81") self.assertEquals(str(train_assets[1]), "example_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_padiw+100:ih+100:50:50") self.assertEquals(str(train_assets[2]), "example_0_3_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81_padiw+288:ih+162:144:81")
def test_read_image_dataset_notyuv(self): dataset_path = VmafConfig.test_resource_path('test_image_dataset_notyuv.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) self.assertEquals(len(assets), 4) self.assertTrue(assets[0].ref_width_height is None) self.assertTrue(assets[0].dis_width_height is None) self.assertEquals(assets[0].quality_width_height, (1920, 1080))
def test_run_vmaf_training_raw_dataset(self): exe = VmafConfig.root_path('run_vmaf_training') cmd = "{exe} {dataset} {param} {param} {output} --parallelize --suppress-plot".format( exe=exe, dataset=self.raw_dataset_filename, param=self.param_filename, output=self.out_model_filepath) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0)
def test_explain_train_test_model(self): model_class = SklearnRandomForestTrainTestModel train_dataset_path = VmafConfig.test_resource_path('test_image_dataset_diffdim.py') train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) fextractor = MomentNorefFeatureExtractor( train_assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, ) fextractor.run(parallelize=True) self.features = fextractor.results xys = model_class.get_xys_from_results(self.features[:7]) model = model_class({'norm_type':'normalize', 'random_state':0}, None) model.train(xys) np.random.seed(0) xs = model_class.get_xs_from_results(self.features[7:]) explainer = LocalExplainer(neighbor_samples=1000) exps = explainer.explain(model, xs) self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4) self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4) self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4) self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4) self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4) self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4) self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4) self.assertEqual(exps['feature_names'], ['Moment_noref_feature_1st_score', 'Moment_noref_feature_2nd_score', 'Moment_noref_feature_var_score'] )
def test_run_vmaf_fextractor_with_cropping(self): # crop_cmd: 288:162:144:81 - crop to 288x162 with upper-left pixel # starting at coordinate (144, 81) ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324, 'crop_cmd': '288:162:144:81', 'quality_width': 288, 'quality_height': 162, }) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324, 'crop_cmd': '288:162:144:81', 'quality_width': 288, 'quality_height': 162, }) self.fextractor = VmafFeatureExtractor([asset, asset_original], None, fifo_mode=False) self.fextractor.run(parallelize=False) results = self.fextractor.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.45365762500000012, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 2.8779373333333331, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9388824973398119, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.942050354166668, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 2.8779373333333331, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.71648420833333, places=4)
class StrredFeatureExtractor(MatlabFeatureExtractor): TYPE = 'STRRED_feature' # VERSION = '1.0' # VERSION = '1.1' # fix matlab code where width and height are mistakenly swapped # VERSION = '1.2' # fix minor frame and prev frame swap issue VERSION = '1.3' # align ST-RRED with ST-RREDopt calculations ATOM_FEATURES = [ 'srred', 'trred', ] DERIVED_ATOM_FEATURES = [ 'strred', ] MATLAB_WORKSPACE = VmafConfig.root_path('matlab', 'strred') @classmethod def _assert_an_asset(cls, asset): super(StrredFeatureExtractor, cls)._assert_an_asset(asset) assert asset.ref_yuv_type == 'yuv420p' and asset.dis_yuv_type == 'yuv420p', \ 'STRRED feature extractor only supports yuv420p for now.' def _generate_result(self, asset): # routine to call the command-line executable and generate quality # scores in the log file. ref_workfile_path = asset.ref_workfile_path dis_workfile_path = asset.dis_workfile_path log_file_path = self._get_log_file_path(asset) current_dir = os.getcwd() + '/' ref_workfile_path = make_absolute_path(ref_workfile_path, current_dir) dis_workfile_path = make_absolute_path(dis_workfile_path, current_dir) log_file_path = make_absolute_path(log_file_path, current_dir) quality_width, quality_height = asset.quality_width_height strred_cmd = '''{matlab} -nodisplay -nosplash -nodesktop -r "run_strred('{ref}', '{dis}', {h}, {w}); exit;" >> {log_file_path}'''.format( matlab=VmafExternalConfig.get_and_assert_matlab(), ref=ref_workfile_path, dis=dis_workfile_path, w=quality_width, h=quality_height, log_file_path=log_file_path, ) if self.logger: self.logger.info(strred_cmd) os.chdir(self.MATLAB_WORKSPACE) run_process(strred_cmd, shell=True) os.chdir(current_dir) @classmethod def _post_process_result(cls, result): # override Executor._post_process_result def _strred(srred_trred): srred, trred = srred_trred try: return srred * trred except TypeError: # possible either srred or trred is None return None result = super(StrredFeatureExtractor, cls)._post_process_result(result) # calculate refvar and disvar from ref1st, ref2nd, dis1st, dis2nd srred_scores_key = cls.get_scores_key('srred') trred_scores_key = cls.get_scores_key('trred') strred_scores_key = cls.get_scores_key('strred') strred_all_same_scores_key = cls.get_scores_key('strred_all_same') srred_scores = result.result_dict[srred_scores_key] trred_scores = result.result_dict[trred_scores_key] assert len(srred_scores) == len(trred_scores) # === Way One: consistent with VMAF framework, which is to multiply S and T scores per frame, then average strred_scores = map(_strred, zip(srred_scores, trred_scores)) # === Way Two: authentic way of calculating STRRED score: average first, then multiply === strred_all_same_scores = ListStats.nonemean( srred_scores) * ListStats.nonemean(trred_scores) * np.ones( len(srred_scores)) result.result_dict[strred_all_same_scores_key] = strred_all_same_scores result.result_dict[strred_scores_key] = strred_scores # validate for feature in cls.DERIVED_ATOM_FEATURES: assert cls.get_scores_key(feature) in result.result_dict return result
dataset_name = 'example' yuv_fmt = 'yuv420p' width = 1920 height = 1080 ref_score = 100.0 from vmaf.config import VmafConfig ref_videos = [ { 'content_id': 0, 'path': VmafConfig.test_resource_path('yuv', 'checkerboard_1920_1080_10_3_0_0.yuv') }, { 'content_id': 1, 'path': VmafConfig.test_resource_path('yuv', 'flat_1920_1080_0.yuv') }, ] dis_videos = [ { 'content_id': 0, 'asset_id': 0, 'os': [100, 100, 100, 100, 100], 'path':
class EnsembleVmafQualityRunner(VmafQualityRunner): TYPE = 'EnsembleVMAF' VERSION = '{}-Ensemble'.format(VmafQualityRunner.VERSION) DEFAULT_MODEL_FILEPATH = [ VmafConfig.model_path("vmaf_v0.6.1.pkl"), VmafConfig.model_path("vmaf_v0.6.1.pkl") ] # this now needs to become a list DEFAULT_FEATURE_DICT = [{ 'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr'] }, { 'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr'] }] def _populate_result_dict(self, feature_result, pred_result, result_dict): result_dict.update(feature_result.result_dict) # add feature result result_dict[self.get_scores_key()] = pred_result[ 'ys_pred'] # add quality score return result_dict def _get_ensemblevmaf_feature_assembler_instance(self, asset): # load TrainTestModel only to retrieve its 'feature_dict' extra info ensem_models = self._load_model(asset) ensemblevmaf_fassemblers = [] for model_ind, model_now in enumerate(ensem_models): feature_dict = model_now.get_appended_info('feature_dict') if feature_dict is None: feature_dict = self.DEFAULT_FEATURE_DICT[model_ind] ensemblevmaf_fassembler = FeatureAssembler( feature_dict=feature_dict, feature_option_dict=None, assets=[asset], logger=self.logger, fifo_mode=self.fifo_mode, delete_workdir=self.delete_workdir, result_store=self.result_store, optional_dict=None, optional_dict2=None, parallelize=False, # parallelization already in a higher level ) ensemblevmaf_fassemblers.append(ensemblevmaf_fassembler) return ensemblevmaf_fassemblers def _get_Nframes(self, pred_result): return len(pred_result['ys_pred']) def _run_on_asset(self, asset): # Override Executor._run_on_asset(self, asset), which runs a # FeatureAssembler, collect a feature vector, run # TrainTestModel.predict() on it, and return a Result object # (in this case, both Executor._run_on_asset(self, asset) and # QualityRunner._read_result(self, asset) get bypassed. ensemblevmaf_fassemblers = self._get_ensemblevmaf_feature_assembler_instance( asset) # each model is associated with a Feature Assembler Nmodels = len(ensemblevmaf_fassemblers) pred_result_ensem_models = [] result_dict = {} for model_ind in range(Nmodels): evmaf_fassembler = ensemblevmaf_fassemblers[model_ind] evmaf_fassembler.run() feature_result = evmaf_fassembler.results[0] model = self._load_model(asset)[model_ind] xs = model.get_per_unit_xs_from_a_result(feature_result) if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict: disable_clip_score = self.optional_dict['disable_clip_score'] else: disable_clip_score = False if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict: enable_transform_score = self.optional_dict[ 'enable_transform_score'] else: enable_transform_score = False pred_result = self.predict_with_model( model, xs, disable_clip_score=disable_clip_score, enable_transform_score=enable_transform_score) result_dict = self._populate_result_dict(feature_result, pred_result, result_dict) pred_result_ensem_models.append(pred_result) assert Nmodels > 0 Nframes = self._get_Nframes(pred_result) all_model_scores = np.zeros((Nmodels, Nframes)) all_model_score_names = self.ensemblevmaf_get_scores_key(Nmodels) for model_ind in range(Nmodels): result_dict[all_model_score_names[ model_ind]] = pred_result_ensem_models[model_ind][ 'ys_pred'] # add quality score all_model_scores[ model_ind, :] = pred_result_ensem_models[model_ind]['ys_pred'] # perform prediction averaging (simple average for now) pred_result_ensem_models_aggregate = np.mean(all_model_scores, axis=0) # write results result_dict[self.get_scores_key()] = pred_result_ensem_models_aggregate return Result(asset, self.executor_id, result_dict) def ensemblevmaf_get_scores_key(self, Nmodels): scores_name_list = [] for model_ind in range(Nmodels): scores_name_list.append(self.TYPE + '_model_' + str(model_ind) + '_scores') return scores_name_list def _load_model(self, asset): if self.optional_dict is not None \ and 'model_filepath' in self.optional_dict \ and self.optional_dict['model_filepath'] is not None: model_filepath = self.optional_dict['model_filepath'] else: model_filepath = self.DEFAULT_MODEL_FILEPATH model = [] for model_filepath_part in model_filepath: model.append( TrainTestModel.from_file(model_filepath_part, self.logger)) return model def _remove_result(self, asset): # Override Executor._remove_result(self, asset) by redirecting it to the # FeatureAssembler. ensemblevmaf_fassemblers = self._get_ensemblevmaf_feature_assembler_instance( asset) for ensemblevmaf_fassembler in ensemblevmaf_fassemblers: ensemblevmaf_fassembler.remove_results()
class STMADFeatureExtractor(MatlabFeatureExtractor): TYPE = "STMAD_feature" VERSION = "0.1" ATOM_FEATURES = ['smad', 'tmad', 'stmad'] DERIVED_ATOM_FEATURES = [ 'smad_all_same', 'tmad_all_same', 'stmad_all_same' ] MATLAB_WORKSPACE = VmafConfig.root_path('matlab', 'STMAD_2011_MatlabCode') # compile necessary functions; need to use mex from within matlab def _custom_init(self): def run_stmad_cmd(stmad_cmd): current_dir = os.getcwd() + '/' os.chdir(self.MATLAB_WORKSPACE) run_process(stmad_cmd, shell=True) os.chdir(current_dir) stmad_mex_cmd_1 = '''{matlab} -nodisplay -nosplash -nodesktop -r "mex ical_std.c; exit;"'''.format( matlab=VmafExternalConfig.get_and_assert_matlab(), ) run_stmad_cmd(stmad_mex_cmd_1) stmad_mex_cmd_2 = '''{matlab} -nodisplay -nosplash -nodesktop -r "mex ical_stat.c; exit;"'''.format( matlab=VmafExternalConfig.get_and_assert_matlab(), ) run_stmad_cmd(stmad_mex_cmd_2) @classmethod def _assert_an_asset(cls, asset): super(STMADFeatureExtractor, cls)._assert_an_asset(asset) assert asset.ref_yuv_type == 'yuv420p' and asset.dis_yuv_type == 'yuv420p', \ 'STMAD feature extractor only supports yuv420p for now.' def _generate_result(self, asset): # routine to call the command-line executable and generate quality # scores in the log file. ref_workfile_path = asset.ref_workfile_path dis_workfile_path = asset.dis_workfile_path log_file_path = self._get_log_file_path(asset) current_dir = os.getcwd() + '/' ref_workfile_path = make_absolute_path(ref_workfile_path, current_dir) dis_workfile_path = make_absolute_path(dis_workfile_path, current_dir) log_file_path = make_absolute_path(log_file_path, current_dir) quality_width, quality_height = asset.quality_width_height stmad_cmd = '''{matlab} -nodisplay -nosplash -nodesktop -r "run_stmad('{ref}', '{dis}', {w}, {h}); exit;" >> {log_file_path}'''.format( matlab=VmafExternalConfig.get_and_assert_matlab(), ref=ref_workfile_path, dis=dis_workfile_path, w=quality_width, h=quality_height, log_file_path=log_file_path, ) if self.logger: self.logger.info(stmad_cmd) os.chdir(self.MATLAB_WORKSPACE) run_process(stmad_cmd, shell=True) os.chdir(current_dir) @classmethod def _post_process_result(cls, result): # override Executor._post_process_result result = super(STMADFeatureExtractor, cls)._post_process_result(result) smad_scores_key = cls.get_scores_key('smad') tmad_scores_key = cls.get_scores_key('tmad') stmad_scores_key = cls.get_scores_key('stmad') smad_all_same_scores_key = cls.get_scores_key('smad_all_same') tmad_all_same_scores_key = cls.get_scores_key('tmad_all_same') stmad_all_same_scores_key = cls.get_scores_key('stmad_all_same') # remember that smad, tmad, stmad are only one value for the whole video so # stmad_all_same_scores is identical to stmad_scores smad_scores = result.result_dict[smad_scores_key] tmad_scores = result.result_dict[tmad_scores_key] stmad_scores = result.result_dict[stmad_scores_key] result.result_dict[smad_scores_key] = smad_scores result.result_dict[tmad_scores_key] = tmad_scores result.result_dict[stmad_scores_key] = stmad_scores result.result_dict[smad_all_same_scores_key] = smad_scores result.result_dict[tmad_all_same_scores_key] = tmad_scores result.result_dict[stmad_all_same_scores_key] = stmad_scores # validate for feature in cls.DERIVED_ATOM_FEATURES: assert cls.get_scores_key(feature) in result.result_dict return result
class VmafLegacyQualityRunner(QualityRunner): TYPE = 'VMAF_legacy' # VERSION = '1.1' VERSION = 'F' + VmafFeatureExtractor.VERSION + '-1.1' FEATURE_ASSEMBLER_DICT = {'VMAF_feature': 'all'} FEATURE_RESCALE_DICT = {'VMAF_feature_vif_scores': (0.0, 1.0), 'VMAF_feature_adm_scores': (0.4, 1.0), 'VMAF_feature_ansnr_scores': (10.0, 50.0), 'VMAF_feature_motion_scores': (0.0, 20.0)} SVM_MODEL_FILE = VmafConfig.model_path("model_V8a.model") # model_v8a.model is trained with customized feature order: SVM_MODEL_ORDERED_SCORES_KEYS = ['VMAF_feature_vif_scores', 'VMAF_feature_adm_scores', 'VMAF_feature_ansnr_scores', 'VMAF_feature_motion_scores'] def _get_quality_scores(self, asset): raise NotImplementedError def _generate_result(self, asset): raise NotImplementedError def _get_vmaf_feature_assembler_instance(self, asset): vmaf_fassembler = FeatureAssembler( feature_dict=self.FEATURE_ASSEMBLER_DICT, feature_option_dict=None, assets=[asset], logger=self.logger, fifo_mode=self.fifo_mode, delete_workdir=self.delete_workdir, result_store=self.result_store, optional_dict=None, optional_dict2=None, parallelize=False, # parallelization already in a higher level ) return vmaf_fassembler def _run_on_asset(self, asset): # Override Executor._run_on_asset(self, asset), which runs a # FeatureAssembler, collect a feature vector, run # TrainTestModel.predict() on it, and return a Result object # (in this case, both Executor._run_on_asset(self, asset) and # QualityRunner._read_result(self, asset) get bypassed. vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset) vmaf_fassembler.run() feature_result = vmaf_fassembler.results[0] # ===================================================================== # SVR predict model = svmutil.svm_load_model(self.SVM_MODEL_FILE) ordered_scaled_scores_list = [] for scores_key in self.SVM_MODEL_ORDERED_SCORES_KEYS: scaled_scores = self._rescale(feature_result[scores_key], self.FEATURE_RESCALE_DICT[scores_key]) ordered_scaled_scores_list.append(scaled_scores) scores = [] for score_vector in zip(*ordered_scaled_scores_list): vif, adm, ansnr, motion = score_vector xs = [[vif, adm, ansnr, motion]] score = svmutil.svm_predict([0], xs, model)[0][0] score = self._post_correction(motion, score) scores.append(score) result_dict = {} # add all feature result result_dict.update(feature_result.result_dict) # add quality score result_dict[self.get_scores_key()] = scores return Result(asset, self.executor_id, result_dict) def _post_correction(self, motion, score): # post-SVM correction if motion > 12.0: val = motion if val > 20.0: val = 20 score *= ((val - 12) * 0.015 + 1) if score > 100.0: score = 100.0 elif score < 0.0: score = 0.0 return score @classmethod def _rescale(cls, vals, lower_upper_bound): lower_bound, upper_bound = lower_upper_bound vals = np.double(vals) vals = np.clip(vals, lower_bound, upper_bound) vals = (vals - lower_bound) / (upper_bound - lower_bound) return vals # override def _remove_result(self, asset): # Override Executor._remove_result by redirecting it to the # FeatureAssembler. vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset) vmaf_fassembler.remove_results()
def setUp(self): self.raw_dataset_filepath = VmafConfig.resource_path("dataset", "NFLX_dataset_public_raw.py") self.derived_dataset_path = VmafConfig.workdir_path("test_derived_dataset.py") self.derived_dataset_path_pyc = VmafConfig.workdir_path("test_derived_dataset.pyc")
def test_load_result_with_none(self): result = FileSystemResultStore.load_result(VmafConfig.test_resource_path('result_with_none.txt')) result.set_score_aggregate_method(ListStats.nonemean) self.assertAlmostEqual(result['STRRED_feature_srred_score'], 5829.2644469999996, places=4)
def main(): if len(sys.argv) < 5: print_usage() return 2 try: q_width = int(sys.argv[1]) q_height = int(sys.argv[2]) ref_file = sys.argv[3] dis_file = sys.argv[4] except ValueError: print_usage() return 2 if q_width < 0 or q_height < 0: print( "quality_width and quality_height must be non-negative, but are {w} and {h}" .format(w=q_width, h=q_height)) print_usage() return 2 model_path = get_cmd_option(sys.argv, 5, len(sys.argv), '--model') out_fmt = get_cmd_option(sys.argv, 5, len(sys.argv), '--out-fmt') if not (out_fmt is None or out_fmt == 'xml' or out_fmt == 'json' or out_fmt == 'text'): print_usage() return 2 ref_fmt = get_cmd_option(sys.argv, 5, len(sys.argv), '--ref-fmt') if not (ref_fmt is None or ref_fmt in FMTS): print('--ref-fmt can only have option among {}'.format( ', '.join(FMTS))) ref_width = get_cmd_option(sys.argv, 5, len(sys.argv), '--ref-width') ref_height = get_cmd_option(sys.argv, 5, len(sys.argv), '--ref-height') dis_width = get_cmd_option(sys.argv, 5, len(sys.argv), '--dis-width') dis_height = get_cmd_option(sys.argv, 5, len(sys.argv), '--dis-height') obj_file = get_cmd_option(sys.argv, 6, len(sys.argv), '--attention') dis_fmt = get_cmd_option(sys.argv, 5, len(sys.argv), '--dis-fmt') if not (dis_fmt is None or dis_fmt in FMTS): print('--dis-fmt can only have option among {}'.format( ', '.join(FMTS))) work_dir = get_cmd_option(sys.argv, 5, len(sys.argv), '--work-dir') pool_method = get_cmd_option(sys.argv, 5, len(sys.argv), '--pool') if not (pool_method is None or pool_method in POOL_METHODS): print('--pool can only have option among {}'.format( ', '.join(POOL_METHODS))) return 2 show_local_explanation = cmd_option_exists(sys.argv, 5, len(sys.argv), '--local-explain') phone_model = cmd_option_exists(sys.argv, 5, len(sys.argv), '--phone-model') enable_conf_interval = cmd_option_exists(sys.argv, 5, len(sys.argv), '--ci') save_plot_dir = get_cmd_option(sys.argv, 5, len(sys.argv), '--save-plot') if work_dir is None: work_dir = VmafConfig.workdir_path() asset_dict = {'quality_width': q_width, 'quality_height': q_height} if ref_fmt is None: asset_dict['ref_yuv_type'] = 'notyuv' else: if ref_width is None or ref_height is None: print( 'if --ref-fmt is specified, both --ref-width and --ref-height must be specified' ) return 2 else: asset_dict['ref_yuv_type'] = ref_fmt asset_dict['ref_width'] = ref_width asset_dict['ref_height'] = ref_height if dis_fmt is None: asset_dict['dis_yuv_type'] = 'notyuv' else: if dis_width is None or dis_height is None: print( 'if --dis-fmt is specified, both --dis-width and --dis-height must be specified' ) return 2 else: asset_dict['dis_yuv_type'] = dis_fmt asset_dict['dis_width'] = dis_width asset_dict['dis_height'] = dis_height if show_local_explanation and enable_conf_interval: print('cannot set both --local-explain and --ci flags') return 2 asset = Asset( dataset="cmd", content_id=abs(hash(get_file_name_without_extension(ref_file))) % (10**16), asset_id=abs(hash(get_file_name_without_extension(ref_file))) % (10**16), workdir_root=work_dir, ref_path=ref_file, dis_path=dis_file, obj_path=obj_file, asset_dict=asset_dict, ) assets = [asset] if show_local_explanation: from vmaf.core.quality_runner_extra import VmafQualityRunnerWithLocalExplainer runner_class = VmafQualityRunnerWithLocalExplainer elif enable_conf_interval: from vmaf.core.quality_runner import BootstrapVmafQualityRunner runner_class = BootstrapVmafQualityRunner else: runner_class = VmafQualityRunner if model_path is None: optional_dict = None else: optional_dict = {'model_filepath': model_path} if phone_model: if optional_dict is None: optional_dict = {} optional_dict['enable_transform_score'] = True runner = runner_class( assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=optional_dict, optional_dict2=None, ) # run runner.run() result = runner.results[0] # pooling if pool_method == 'harmonic_mean': result.set_score_aggregate_method(ListStats.harmonic_mean) elif pool_method == 'min': result.set_score_aggregate_method(np.min) elif pool_method == 'median': result.set_score_aggregate_method(np.median) elif pool_method == 'perc5': result.set_score_aggregate_method(ListStats.perc5) elif pool_method == 'perc10': result.set_score_aggregate_method(ListStats.perc10) elif pool_method == 'perc20': result.set_score_aggregate_method(ListStats.perc20) else: # None or 'mean' pass # output if out_fmt == 'xml': print(result.to_xml()) elif out_fmt == 'json': print(result.to_json()) else: # None or 'text' print(str(result)) # local explanation if show_local_explanation: runner.show_local_explanations([result]) if save_plot_dir is None: DisplayConfig.show() else: DisplayConfig.show(write_to_dir=save_plot_dir) return 0
def setUp(self): self.output_model_filepath = VmafConfig.workspace_path( "model", "test_output_model.pkl")
def read_dataset(dataset, **kwargs): groundtruth_key = kwargs[ 'groundtruth_key'] if 'groundtruth_key' in kwargs else None skip_asset_with_none_groundtruth = kwargs['skip_asset_with_none_groundtruth'] \ if 'skip_asset_with_none_groundtruth' in kwargs else False # asserts, can add more to the list... assert hasattr(dataset, 'dataset_name') assert hasattr(dataset, 'ref_videos') assert hasattr(dataset, 'dis_videos') assert hasattr(dataset, 'yuv_fmt') or all( ['yuv_fmt' in ref_video for ref_video in dataset.ref_videos]) data_set_name = dataset.dataset_name ref_videos = dataset.ref_videos dis_videos = dataset.dis_videos width = dataset.width if hasattr(dataset, 'width') else None height = dataset.height if hasattr(dataset, 'height') else None yuv_fmt = dataset.yuv_fmt if hasattr(dataset, 'yuv_fmt') else None quality_width = dataset.quality_width if hasattr(dataset, 'quality_width') else None quality_height = dataset.quality_height if hasattr( dataset, 'quality_height') else None resampling_type = dataset.resampling_type if hasattr( dataset, 'resampling_type') else None crop_cmd = dataset.crop_cmd if hasattr(dataset, 'crop_cmd') else None pad_cmd = dataset.pad_cmd if hasattr(dataset, 'pad_cmd') else None workfile_yuv_type = dataset.workfile_yuv_type if hasattr( dataset, 'workfile_yuv_type') else None duration_sec = dataset.duration_sec if hasattr(dataset, 'duration_sec') else None ref_dict = {} # dictionary of content_id -> path for ref videos for ref_video in ref_videos: ref_dict[ref_video['content_id']] = ref_video assets = [] for dis_video in dis_videos: if groundtruth_key is not None: groundtruth = dis_video[groundtruth_key] else: if 'dmos' in dis_video: groundtruth = dis_video['dmos'] elif 'mos' in dis_video: groundtruth = dis_video['mos'] elif 'groundtruth' in dis_video: groundtruth = dis_video['groundtruth'] else: groundtruth = None if 'os' in dis_video: raw_groundtruth = dis_video['os'] else: raw_groundtruth = None if 'groundtruth_std' in dis_video: groundtruth_std = dis_video['groundtruth_std'] else: groundtruth_std = None if 'fps' in dis_video: fps = dis_video['fps'] else: fps = None if 'rebuf_indices' in dis_video: rebuf_indices = dis_video['rebuf_indices'] else: rebuf_indices = None ref_video = ref_dict[dis_video['content_id']] ref_path = ref_video['path'] ref_yuv_fmt_ = yuv_fmt if yuv_fmt is not None else ref_dict[ dis_video['content_id']]['yuv_fmt'] dis_yuv_fmt_ = dis_video[ 'yuv_fmt'] if 'yuv_fmt' in dis_video else ref_yuv_fmt_ if width is not None: width_ = width elif 'width' in ref_video: width_ = ref_video[ 'width'] # NOTE: width in ref_video not dis_video else: width_ = None if height is not None: height_ = height elif 'height' in ref_video: height_ = ref_video[ 'height'] # NOTE: height in ref_video not dis_video else: height_ = None if quality_width is not None: quality_width_ = quality_width elif 'quality_width' in dis_video: quality_width_ = dis_video['quality_width'] else: quality_width_ = None if quality_height is not None: quality_height_ = quality_height elif 'quality_height' in dis_video: quality_height_ = dis_video['quality_height'] else: quality_height_ = None if resampling_type is not None: resampling_type_ = resampling_type elif 'resampling_type' in dis_video: resampling_type_ = dis_video['resampling_type'] else: resampling_type_ = None if crop_cmd is not None: crop_cmd_ = crop_cmd elif 'crop_cmd' in dis_video: crop_cmd_ = dis_video['crop_cmd'] else: crop_cmd_ = None if pad_cmd is not None: pad_cmd_ = pad_cmd elif 'pad_cmd' in dis_video: pad_cmd_ = dis_video['pad_cmd'] else: pad_cmd_ = None if duration_sec is not None: duration_sec_ = duration_sec elif 'duration_sec' in dis_video: duration_sec_ = dis_video['duration_sec'] else: duration_sec_ = None asset_dict = { 'ref_yuv_type': ref_yuv_fmt_, 'dis_yuv_type': dis_yuv_fmt_ } if width_ is not None: if asset_dict['ref_yuv_type'] != 'notyuv': asset_dict['ref_width'] = width_ if asset_dict['dis_yuv_type'] != 'notyuv': asset_dict['dis_width'] = width_ if height_ is not None: if asset_dict['ref_yuv_type'] != 'notyuv': asset_dict['ref_height'] = height_ if asset_dict['dis_yuv_type'] != 'notyuv': asset_dict['dis_height'] = height_ if groundtruth is not None: asset_dict['groundtruth'] = groundtruth if raw_groundtruth is not None: asset_dict['raw_groundtruth'] = raw_groundtruth if groundtruth_std is not None: asset_dict['groundtruth_std'] = groundtruth_std if quality_width_ is not None: asset_dict['quality_width'] = quality_width_ if quality_height_ is not None: asset_dict['quality_height'] = quality_height_ if resampling_type_ is not None: asset_dict['resampling_type'] = resampling_type_ if crop_cmd_ is not None: asset_dict['crop_cmd'] = crop_cmd_ if pad_cmd_ is not None: asset_dict['pad_cmd'] = pad_cmd_ if duration_sec_ is not None: asset_dict['duration_sec'] = duration_sec_ if workfile_yuv_type is not None: asset_dict['workfile_yuv_type'] = workfile_yuv_type if fps is not None: asset_dict['fps'] = fps if rebuf_indices is not None: asset_dict['rebuf_indices'] = rebuf_indices if groundtruth is None and skip_asset_with_none_groundtruth: pass else: asset = Asset( dataset=data_set_name, content_id=dis_video['content_id'], asset_id=dis_video['asset_id'], workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_video['path'], asset_dict=asset_dict, ) assets.append(asset) return assets
def test_run_testlibdyn_runner(self): print('test on running TESTLIBDYN runner...') ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) self.runner = LibDynRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale0_score'], 0.363420458333, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale1_score'], 0.766647520833, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale2_score'], 0.862854708333, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale3_score'], 0.915971791667, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_adm2_score'], 0.93458777083333333, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_psnr_score'], 30.7550666667, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale1_score'], 0.999999958333, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale2_score'], 0.999999416667, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale3_score'], 0.999999208333, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_psnr_score'], 60.0, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['TESTLIBDYN_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['TESTLIBDYN_score'], 76.699271272486044, places=3) self.assertAlmostEqual(results[1]['TESTLIBDYN_score'], 99.946416604585025, places=4)
def test_run_vmaf_fextractor_with_cropping_and_padding_to_original_wh_proc( self): # crop_cmd: 288:162:144:81 - crop to the center 288x162 image # pad_cmd: iw+288:ih+162:144:81 - pad back to the original size ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324, 'crop_cmd': '288:162:144:81', 'pad_cmd': 'iw+288:ih+162:144:81', 'quality_width': 576, 'quality_height': 324, 'ref_proc_callback': 'identity', 'dis_proc_callback': 'identity', }) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324, 'crop_cmd': '288:162:144:81', 'pad_cmd': 'iw+288:ih+162:144:81', 'quality_width': 576, 'quality_height': 324, }) self.fextractor = VmafFeatureExtractor([asset, asset_original], None, fifo_mode=True) self.fextractor.run(parallelize=True) results = self.fextractor.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.64106379166666672, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 0.7203213958333331, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9469305256822512, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 32.78451041666667, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 0.7203213958333331, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 40.280504208333333, places=4)
def test_run_vmaf_fextractor_with_padding(self): # pad_cmd: iw+100:ih+100:50:50 - pad to (iw+100)x(ih+100), where iw is # input width, ih is input height, and starting point is (-50, -50) ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324, 'pad_cmd': 'iw+100:ih+100:50:50', 'quality_width': 676, 'quality_height': 424, }) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324, 'pad_cmd': 'iw+100:ih+100:50:50', 'quality_width': 676, 'quality_height': 424, }) self.fextractor = VmafFeatureExtractor([asset, asset_original], None, fifo_mode=True) self.fextractor.run(parallelize=False) results = self.fextractor.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.51023564583333325, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 2.6397702083333332, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9410537302204777, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 26.893242291666667, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 2.6397702083333332, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 34.306043416666668, places=4)
class VmafossExecQualityRunner(QualityRunner): TYPE = 'VMAFOSSEXEC' # VERSION = '0.3' # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv3.pkl.model") # VERSION = '0.3.1' # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv3a.pkl.model") # VERSION = '0.3.2' # ALGO_VERSION = 0 # # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv4.pkl.model") # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv4.pkl") VERSION = 'F' + VmafFeatureExtractor.VERSION + '-0.6.1' ALGO_VERSION = 2 # trained with resource/param/vmaf_v6.py on private/user/zli/resource/dataset/dataset/derived/vmafplusstudy_laptop_raw_generalandcornercase.py, MLER, y=x+17 DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_v0.6.1.pkl") FEATURES = ['adm2', 'adm_scale0', 'adm_scale1', 'adm_scale2', 'adm_scale3', 'motion', 'vif_scale0', 'vif_scale1', 'vif_scale2', 'vif_scale3', 'vif', 'psnr', 'ssim', 'ms_ssim', 'motion2'] @classmethod def get_feature_scores_key(cls, atom_feature): return "{type}_{atom_feature}_scores".format( type=cls.TYPE, atom_feature=atom_feature) def _generate_result(self, asset): # routine to call the command-line executable and generate quality # scores in the log file. log_file_path = self._get_log_file_path(asset) if self.optional_dict is not None \ and 'model_filepath' in self.optional_dict \ and self.optional_dict['model_filepath'] is not None: model_filepath = self.optional_dict['model_filepath'] else: model_filepath = self.DEFAULT_MODEL_FILEPATH if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict: disable_clip_score = self.optional_dict['disable_clip_score'] else: disable_clip_score = False if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict: enable_transform_score = self.optional_dict['enable_transform_score'] else: enable_transform_score = False if self.optional_dict is not None and 'phone_model' in self.optional_dict: phone_model = self.optional_dict['phone_model'] else: phone_model = False if self.optional_dict is not None and 'disable_avx' in self.optional_dict: disable_avx = self.optional_dict['disable_avx'] else: disable_avx = False quality_width, quality_height = asset.quality_width_height fmt=self._get_workfile_yuv_type(asset) w=quality_width h=quality_height ref_path=asset.ref_workfile_path dis_path=asset.dis_workfile_path model=model_filepath exe=self._get_exec() logger = self.logger ExternalProgramCaller.call_vmafossexec(fmt, w, h, ref_path, dis_path, model, log_file_path, disable_clip_score, enable_transform_score, phone_model, disable_avx, exe, logger) def _get_exec(self): return None # signaling default def _get_quality_scores(self, asset): # routine to read the quality scores from the log file, and return # the scores in a dictionary format. log_file_path = self._get_log_file_path(asset) tree = ElementTree.parse(log_file_path) root = tree.getroot() scores = [] feature_scores = [[] for _ in self.FEATURES] for frame in root.findall('frames/frame'): scores.append(float(frame.attrib['vmaf'])) for i_feature, feature in enumerate(self.FEATURES): try: feature_scores[i_feature].append(float(frame.attrib[feature])) except KeyError: pass # some features may be missing assert len(scores) != 0 quality_result = { self.get_scores_key(): scores, } for i_feature, feature in enumerate(self.FEATURES): if len(feature_scores[i_feature]) != 0: quality_result[self.get_feature_scores_key(feature)] = feature_scores[i_feature] return quality_result
from vmaf.config import VmafConfig dataset_name = 'test_image' yuv_fmt = 'yuv444p' dataset_dir = VmafConfig.test_resource_path('test_image_yuv') quality_width = 200 quality_height = 100 resampling_type = 'bicubic' ref_videos = [ {'content_id': 0, 'content_name': '100007', 'height': 321, 'path': dataset_dir + '/100007.yuv', 'width': 481}, {'content_id': 1, 'content_name': '100039', 'height': 321, 'path': dataset_dir + '/100039.yuv', 'width': 481}, {'content_id': 2, 'content_name': '100075', 'height': 321, 'path': dataset_dir + '/100075.yuv', 'width': 481}, {'content_id': 4, 'content_name': '100098', 'height': 321, 'path': dataset_dir + '/100098.yuv',
class VmafQualityRunner(QualityRunner): TYPE = 'VMAF' # VERSION = '0.1' # using model nflxall_vmafv1.pkl, VmafFeatureExtractor VERSION 0.1 # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv1.pkl") # trained with resource/param/vmaf_v1.py on private/resource/dataset/NFLX_dataset.py (30 subjects) # VERSION = '0.2' # using model nflxall_vmafv2.pkl, VmafFeatureExtractor VERSION 0.2.1 # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv2.pkl") # trained with resource/param/vmaf_v2.py on private/resource/dataset/NFLX_dataset.py (30 subjects) # VERSION = '0.3' # using model nflxall_vmafv3.pkl, VmafFeatureExtractor VERSION 0.2.1 # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv3.pkl") # trained with resource/param/vmaf_v3.py on private/resource/dataset/NFLX_dataset.py (30 subjects) # VERSION = '0.3.1' # using model nflxall_vmafv3.pkl, VmafFeatureExtractor VERSION 0.2.1, NFLX_dataset with 26 subjects (last 4 outliers removed) # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv3a.pkl") # trained with resource/param/vmaf_v3.py on private/resource/dataset/NFLX_dataset.py (26 subjects) # VERSION = '0.3.2' # using model nflxall_vmafv4.pkl, VmafFeatureExtractor VERSION 0.2.2, NFLX_dataset with 26 subjects (last 4 outliers removed) # ALGO_VERSION = 0 # DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv4.pkl") # trained with resource/param/vmaf_v4.py on private/resource/dataset/NFLX_dataset.py (26 subjects) # using model vmaf_v0.6.1.pkl. VmafFeatureExtractor VERSION 0.2.4b. VERSION = 'F' + VmafFeatureExtractor.VERSION + '-0.6.1' ALGO_VERSION = 2 # trained with resource/param/vmaf_v6.py on private/user/zli/resource/dataset/dataset/derived/vmafplusstudy_laptop_raw_generalandcornercase.py, MLER, y=x+17 DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_v0.6.1.pkl") DEFAULT_FEATURE_DICT = {'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']} # for backward-compatible with older model only def _get_quality_scores(self, asset): raise NotImplementedError def _generate_result(self, asset): raise NotImplementedError def _get_vmaf_feature_assembler_instance(self, asset): # load TrainTestModel only to retrieve its 'feature_dict' extra info feature_dict = self._load_model(asset).get_appended_info('feature_dict') if feature_dict is None: feature_dict = self.DEFAULT_FEATURE_DICT vmaf_fassembler = FeatureAssembler( feature_dict=feature_dict, feature_option_dict=None, assets=[asset], logger=self.logger, fifo_mode=self.fifo_mode, delete_workdir=self.delete_workdir, result_store=self.result_store, optional_dict=None, optional_dict2=None, parallelize=False, # parallelization already in a higher level ) return vmaf_fassembler def _run_on_asset(self, asset): # Override Executor._run_on_asset(self, asset), which runs a # FeatureAssembler, collect a feature vector, run # TrainTestModel.predict() on it, and return a Result object # (in this case, both Executor._run_on_asset(self, asset) and # QualityRunner._read_result(self, asset) get bypassed. vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset) vmaf_fassembler.run() feature_result = vmaf_fassembler.results[0] model = self._load_model(asset) xs = model.get_per_unit_xs_from_a_result(feature_result) if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict: disable_clip_score = self.optional_dict['disable_clip_score'] else: disable_clip_score = False if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict: enable_transform_score = self.optional_dict['enable_transform_score'] else: enable_transform_score = False pred_result = self.predict_with_model(model, xs, disable_clip_score=disable_clip_score, enable_transform_score=enable_transform_score) result_dict = self._populate_result_dict(feature_result, pred_result) return Result(asset, self.executor_id, result_dict) def _populate_result_dict(self, feature_result, pred_result): result_dict = {} result_dict.update(feature_result.result_dict) # add feature result result_dict[self.get_scores_key()] = pred_result['ys_pred'] # add quality score return result_dict @classmethod def predict_with_model(cls, model, xs, **kwargs): ys_pred = model.predict(xs)['ys_label_pred'] do_transform_score = cls._do_transform_score(kwargs) if do_transform_score: ys_pred = cls.transform_score(model, ys_pred) else: pass if 'disable_clip_score' in kwargs and kwargs['disable_clip_score'] is True: pass else: ys_pred = cls.clip_score(model, ys_pred) return {'ys_pred': ys_pred} @staticmethod def _do_transform_score(kwargs): return 'enable_transform_score' in kwargs and kwargs['enable_transform_score'] is True @staticmethod def set_transform_score(model, score_transform): model.append_info('score_transform', score_transform) @staticmethod def set_clip_score(model, score_clip): model.append_info('score_clip', score_clip) @staticmethod def transform_score(model, ys_pred): """ Do post processing: transform final quality score e.g. via polynomial {'p0': 1, 'p1': 1, 'p2': 0.5} means transform through 1 + x + 0.5 * x^2. For now, only support polynomail up to 2nd-order. """ transform_dict = model.get_appended_info('score_transform') if transform_dict is None: return ys_pred y_in = ys_pred y_out = np.zeros(ys_pred.shape) # quadratic transform if 'p0' in transform_dict: y_out += transform_dict['p0'] if 'p1' in transform_dict: y_out += transform_dict['p1'] * y_in if 'p2' in transform_dict: y_out += transform_dict['p2'] * y_in * y_in # rectification if 'out_lte_in' in transform_dict and transform_dict['out_lte_in'] == 'true': # output must be less than or equal to input y_out = np.minimum(y_out, y_in) if 'out_gte_in' in transform_dict and transform_dict['out_gte_in'] == 'true': # output must be greater than or equal to input y_out = np.maximum(y_out, y_in) return y_out @staticmethod def clip_score(model, ys_pred): """ Do post processing: clip final quality score within e.g. [0, 100] """ score_clip = model.get_appended_info('score_clip') if score_clip is not None: lb, ub = score_clip ys_pred = np.clip(ys_pred, lb, ub) return ys_pred def _load_model(self, asset): if self.optional_dict is not None \ and 'model_filepath' in self.optional_dict \ and self.optional_dict['model_filepath'] is not None: model_filepath = self.optional_dict['model_filepath'] else: model_filepath = self.DEFAULT_MODEL_FILEPATH train_test_model_class = self.get_train_test_model_class() model = train_test_model_class.from_file(model_filepath, self.logger) return model def get_train_test_model_class(self): return LibsvmNusvrTrainTestModel def _remove_result(self, asset): # Override Executor._remove_result(self, asset) by redirecting it to the # FeatureAssembler. vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset) vmaf_fassembler.remove_results()
def test_run_vmafrc_runner_yuv422p10le(self): ref_path = VmafConfig.test_resource_path( "yuv", "src01_hrc00_576x324.yuv422p10le.yuv") dis_path = VmafConfig.test_resource_path( "yuv", "src01_hrc01_576x324.yuv422p10le.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324, 'yuv_type': 'yuv422p10le' }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324, 'yuv_type': 'yuv422p10le' }) self.runner = VmafrcQualityRunner([asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'psnr': True, 'ssim': True, 'ms_ssim': True, }) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFRC_vif_scale0_score'], 0.3634208125, places=4) self.assertAlmostEqual(results[0]['VMAFRC_vif_scale1_score'], 0.7666474166666667, places=4) self.assertAlmostEqual(results[0]['VMAFRC_vif_scale2_score'], 0.8628533333333334, places=4) self.assertAlmostEqual(results[0]['VMAFRC_vif_scale3_score'], 0.9159719583333334, places=4) self.assertAlmostEqual(results[0]['VMAFRC_motion2_score'], 3.895352291666667, places=4) self.assertAlmostEqual(results[0]['VMAFRC_adm2_score'], 0.9345877291666667, places=4) self.assertAlmostEqual(results[0]['VMAFRC_float_psnr_score'], 30.780577083333331, places=4) self.assertAlmostEqual(results[0]['VMAFRC_float_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['VMAFRC_float_ms_ssim_score'], 0.9632406874999999, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale1_score'], 0.9999998541666666, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale2_score'], 0.9999996041666667, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale3_score'], 0.9999991458333334, places=4) self.assertAlmostEqual(results[1]['VMAFRC_motion2_score'], 3.895352291666667, places=4) self.assertAlmostEqual(results[1]['VMAFRC_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_float_psnr_score'], 72.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_float_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_float_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['VMAFRC_score'], 76.69926875, places=4) self.assertAlmostEqual(results[1]['VMAFRC_score'], 99.94641666666666, places=4)
def test_read_dataset_fps_bad_rebuf_indices(self): train_dataset_path = VmafConfig.test_resource_path('test_dataset_fps_bad_rebufinds.py') train_dataset = import_python_file(train_dataset_path) with self.assertRaises(AssertionError): train_assets = read_dataset(train_dataset)
def test_run_vmafrc_runner_yuv420p10le_sparks(self): ref_path = VmafConfig.test_resource_path( "yuv", "sparks_ref_480x270.yuv42010le.yuv") dis_path = VmafConfig.test_resource_path( "yuv", "sparks_dis_480x270.yuv42010le.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 480, 'height': 270, 'yuv_type': 'yuv420p10le' }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 480, 'height': 270, 'yuv_type': 'yuv420p10le' }) self.runner = VmafrcQualityRunner([asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'psnr': True, 'ssim': True, 'ms_ssim': True, }) self.runner.run(parallelize=True) results = self.runner.results self.assertAlmostEqual(results[0]['VMAFRC_vif_scale0_score'], 0.9240746, places=4) self.assertAlmostEqual(results[0]['VMAFRC_vif_scale1_score'], 0.9968371999999999, places=4) self.assertAlmostEqual(results[0]['VMAFRC_vif_scale2_score'], 0.9987575999999999, places=4) self.assertAlmostEqual(results[0]['VMAFRC_vif_scale3_score'], 0.9993221999999999, places=4) self.assertAlmostEqual(results[0]['VMAFRC_motion2_score'], 0.7523685999999999, places=4) self.assertAlmostEqual(results[0]['VMAFRC_adm2_score'], 0.9981770000000001, places=4) self.assertAlmostEqual(results[0]['VMAFRC_float_psnr_score'], 48.81622, places=4) self.assertAlmostEqual(results[0]['VMAFRC_float_ssim_score'], 0.99566, places=4) self.assertAlmostEqual(results[0]['VMAFRC_float_ms_ssim_score'], 0.9993778000000001, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale1_score'], 0.9999990000000001, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale2_score'], 0.9999990000000001, places=4) self.assertAlmostEqual(results[1]['VMAFRC_vif_scale3_score'], 0.9999990000000001, places=4) self.assertAlmostEqual(results[1]['VMAFRC_motion2_score'], 0.7523685999999999, places=4) self.assertAlmostEqual(results[1]['VMAFRC_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_float_psnr_score'], 72.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_float_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFRC_float_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['VMAFRC_score'], 97.90585999999999, places=4) self.assertAlmostEqual(results[1]['VMAFRC_score'], 98.47138, places=4)
class BootstrapVmafQualityRunner(VmafQualityRunner): TYPE = "BOOTSTRAP_VMAF" VERSION = VmafQualityRunner.VERSION + '-' + 'M' + BootstrapLibsvmNusvrTrainTestModel.VERSION ALGO_VERSION = None # "vmaf_b_v0.6.3": plain bootstrapping, "vmaf_rb_v0.6.3": residue bootstrapping DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_b_v0.6.3", "vmaf_b_v0.6.3.pkl") def _populate_result_dict(self, feature_result, pred_result): result_dict = {} result_dict.update(feature_result.result_dict) # add feature result result_dict[self.get_scores_key()] = pred_result[ 'ys_pred'] # add quality score result_dict[self.get_all_models_scores_key()] = pred_result[ 'ys_pred_all_models'] # add quality score from all models result_dict[self.get_bagging_scores_key()] = pred_result[ 'ys_pred_bagging'] # add bagging quality score result_dict[self.get_stddev_scores_key()] = pred_result[ 'ys_pred_stddev'] # add stddev of bootstrapped quality score result_dict[self.get_ci95_low_scores_key()] = pred_result[ 'ys_pred_ci95_low'] # add ci95 of bootstrapped quality score result_dict[self.get_ci95_high_scores_key()] = pred_result[ 'ys_pred_ci95_high'] # add ci95 of bootstrapped quality score return result_dict @classmethod def predict_with_model(cls, model, xs, **kwargs): DELTA = 1e-2 result = model.predict(xs) ys_pred_all_models = result['ys_label_pred_all_models'] ys_pred = result['ys_label_pred'] ys_pred_bagging = result['ys_label_pred_bagging'] ys_pred_stddev = result['ys_label_pred_stddev'] ys_pred_ci95_low = result['ys_label_pred_ci95_low'] ys_pred_ci95_high = result['ys_label_pred_ci95_high'] ys_pred_plus = ys_pred_bagging + DELTA ys_pred_minus = ys_pred_bagging - DELTA do_transform_score = cls._do_transform_score(kwargs) if do_transform_score: ys_pred_all_models = np.array([ cls.transform_score(model, ys_pred_some_model) for ys_pred_some_model in ys_pred_all_models ]) ys_pred = cls.transform_score(model, ys_pred) ys_pred_bagging = cls.transform_score(model, ys_pred_bagging) ys_pred_plus = cls.transform_score(model, ys_pred_plus) ys_pred_minus = cls.transform_score(model, ys_pred_minus) ys_pred_ci95_low = cls.transform_score(model, ys_pred_ci95_low) ys_pred_ci95_high = cls.transform_score(model, ys_pred_ci95_high) else: pass if 'disable_clip_score' in kwargs and kwargs[ 'disable_clip_score'] is True: pass else: ys_pred_all_models = np.array([ cls.clip_score(model, ys_pred_some_model) for ys_pred_some_model in ys_pred_all_models ]) ys_pred = cls.clip_score(model, ys_pred) ys_pred_bagging = cls.clip_score(model, ys_pred_bagging) ys_pred_plus = cls.clip_score(model, ys_pred_plus) ys_pred_minus = cls.clip_score(model, ys_pred_minus) ys_pred_ci95_low = cls.clip_score(model, ys_pred_ci95_low) ys_pred_ci95_high = cls.clip_score(model, ys_pred_ci95_high) # stddev score transform is applied after transform, clip, or both, or neither slope = ((ys_pred_plus - ys_pred_minus) / (2.0 * DELTA)) ys_pred_stddev = ys_pred_stddev * slope return { 'ys_pred_all_models': ys_pred_all_models, 'ys_pred': ys_pred, 'ys_pred_bagging': ys_pred_bagging, 'ys_pred_stddev': ys_pred_stddev, 'ys_pred_ci95_low': ys_pred_ci95_low, 'ys_pred_ci95_high': ys_pred_ci95_high, } def get_train_test_model_class(self): # overide VmafQualityRunner.get_train_test_model_class return BootstrapLibsvmNusvrTrainTestModel @classmethod def get_all_models_scores_key(cls): return cls.TYPE + '_all_models_scores' @classmethod def get_all_models_score_key(cls): return cls.TYPE + '_all_models_score' @classmethod def get_bagging_scores_key(cls): return cls.TYPE + '_bagging_scores' @classmethod def get_bagging_score_key(cls): return cls.TYPE + '_bagging_score' @classmethod def get_stddev_scores_key(cls): return cls.TYPE + '_stddev_scores' @classmethod def get_stddev_score_key(cls): return cls.TYPE + '_stddev_score' @classmethod def get_ci95_low_scores_key(cls): return cls.TYPE + '_ci95_low_scores' @classmethod def get_ci95_low_score_key(cls): return cls.TYPE + '_ci95_low_score' @classmethod def get_ci95_high_scores_key(cls): return cls.TYPE + '_ci95_high_scores' @classmethod def get_ci95_high_score_key(cls): return cls.TYPE + '_ci95_high_score'
def test_run_vmafrc_compare_directly_with_ossexec_420_10bit(self): ref_path = VmafConfig.test_resource_path( "yuv", "sparks_ref_480x270.yuv42010le.yuv") dis_path = VmafConfig.test_resource_path( "yuv", "sparks_dis_480x270.yuv42010le.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 480, 'height': 270, 'yuv_type': 'yuv420p10le' }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 480, 'height': 270, 'yuv_type': 'yuv420p10le' }) self.runner = VmafrcQualityRunner([asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'psnr': True, 'ssim': True, 'ms_ssim': True, }) self.runner.run(parallelize=False) results_rc = self.runner.results self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, ) self.runner.run(parallelize=False) results = self.runner.results self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale0_score'], results[0]['VMAFOSSEXEC_vif_scale0_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale1_score'], results[0]['VMAFOSSEXEC_vif_scale1_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale2_score'], results[0]['VMAFOSSEXEC_vif_scale2_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale3_score'], results[0]['VMAFOSSEXEC_vif_scale3_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_motion2_score'], results[0]['VMAFOSSEXEC_motion2_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_adm2_score'], results[0]['VMAFOSSEXEC_adm2_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_float_psnr_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) self.assertAlmostEqual(results_rc[0]['VMAFRC_float_ssim_score'], results[0]['VMAFOSSEXEC_ssim_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_float_ms_ssim_score'], results[0]['VMAFOSSEXEC_ms_ssim_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale0_score'], results[1]['VMAFOSSEXEC_vif_scale0_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale1_score'], results[1]['VMAFOSSEXEC_vif_scale1_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale2_score'], results[1]['VMAFOSSEXEC_vif_scale2_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale3_score'], results[1]['VMAFOSSEXEC_vif_scale3_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_motion2_score'], results[1]['VMAFOSSEXEC_motion2_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_adm2_score'], results[1]['VMAFOSSEXEC_adm2_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_float_psnr_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) self.assertAlmostEqual(results_rc[1]['VMAFRC_float_ssim_score'], results[1]['VMAFOSSEXEC_ssim_score'], places=5) self.assertAlmostEqual(results_rc[1]['VMAFRC_float_ms_ssim_score'], results[1]['VMAFOSSEXEC_ms_ssim_score'], places=5) self.assertAlmostEqual(results_rc[0]['VMAFRC_score'], results[0]['VMAFOSSEXEC_score'], places=4) self.assertAlmostEqual(results_rc[1]['VMAFRC_score'], results[1]['VMAFOSSEXEC_score'], places=4)
class NiqeQualityRunner(QualityRunner): TYPE = 'NIQE' # VERSION = '0.1' VERSION = '0.2' # update model to py3 DEFAULT_MODEL_FILEPATH = VmafConfig.model_path('other_models', 'niqe_v0.1.pkl') DEFAULT_FEATURE_DICT = {'NIQE_noref_feature': 'all'} def _get_quality_scores(self, asset): raise NotImplementedError def _generate_result(self, asset): raise NotImplementedError def _get_niqe_feature_assembler_instance(self, asset): # load TrainTestModel only to retrieve its 'feature_dict' extra info model = self._load_model(asset) # need this so that FeatureAssembler can find NiqeNorefFeatureExtractor: from vmaf.core.noref_feature_extractor import NiqeNorefFeatureExtractor feature_dict = model.get_appended_info('feature_dict') if feature_dict is None: feature_dict = self.DEFAULT_FEATURE_DICT feature_optional_dict = model.get_appended_info( 'feature_optional_dict') vmaf_fassembler = FeatureAssembler( feature_dict=feature_dict, feature_option_dict=None, assets=[asset], logger=self.logger, fifo_mode=self.fifo_mode, delete_workdir=self.delete_workdir, result_store=self.result_store, optional_dict=feature_optional_dict, optional_dict2=None, parallelize=False, ) return vmaf_fassembler def _load_model(self, asset): if self.optional_dict is not None \ and 'model_filepath' in self.optional_dict \ and self.optional_dict['model_filepath'] is not None: model_filepath = self.optional_dict['model_filepath'] else: model_filepath = self.DEFAULT_MODEL_FILEPATH model = TrainTestModel.from_file(model_filepath, self.logger) return model def _run_on_asset(self, asset): # Override Executor._run_on_asset(self, asset), which runs a # FeatureAssembler, collect a feature vector, run # TrainTestModel.predict() on it, and return a Result object # (in this case, both Executor._run_on_asset(self, asset) and # QualityRunner._read_result(self, asset) get bypassed. niqe_fassembler = self._get_niqe_feature_assembler_instance(asset) niqe_fassembler.run() feature_result = niqe_fassembler.results[0] # xs = NiqeTrainTestModel.get_perframe_xs_from_result(feature_result) xs = NiqeTrainTestModel.get_xs_from_results([feature_result]) model = self._load_model(asset) ys_pred = model.predict(xs)['ys_label_pred'] result_dict = {} # add all feature result result_dict.update(feature_result.result_dict) # add quality score result_dict[self.get_scores_key()] = ys_pred return Result(asset, self.executor_id, result_dict) def _remove_result(self, asset): # Override Executor._remove_result(self, asset) by redirecting it to the # FeatureAssembler. vmaf_fassembler = self._get_niqe_feature_assembler_instance(asset) vmaf_fassembler.remove_results()
__copyright__ = "Copyright 2016-2018, Netflix, Inc." __license__ = "Apache, Version 2.0" import matplotlib.pyplot as plt import numpy as np from vmaf.config import VmafConfig, DisplayConfig from vmaf.routine import run_vmaf_cv, run_vmaf_kfold_cv if __name__ == '__main__': # ==== Run simple cross validation: one training and one testing dataset ==== run_vmaf_cv( train_dataset_filepath=VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py'), test_dataset_filepath=VmafConfig.resource_path('dataset', 'VQEGHD3_dataset.py'), param_filepath=VmafConfig.resource_path('param', 'vmaf_v3.py'), output_model_filepath=VmafConfig.workspace_path('model', 'test_model1.pkl'), ) # ==== Run cross validation across genres (tough test) ==== nflx_dataset_path = VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py') contentid_groups = [ [0, 5], # cartoon: BigBuckBunny, FoxBird [1], # CG: BirdsInCage [2, 6, 7], # complex: CrowdRun, OldTownCross, Seeking [3, 4], # ElFuente: ElFuente1, ElFuente2 [8], # sports: Tennis ] param_filepath = VmafConfig.resource_path('param', 'vmaf_v3.py')
class SpEEDMatlabFeatureExtractor(MatlabFeatureExtractor): TYPE = 'SpEED_Matlab_feature' VERSION = '0.1' scale_list = [2, 3, 4] ATOM_FEATURES = [] DERIVED_ATOM_FEATURES = [] for scale_now in scale_list: ATOM_FEATURES.append('sspeed_' + str(scale_now)) ATOM_FEATURES.append('tspeed_' + str(scale_now)) DERIVED_ATOM_FEATURES.append('speed_' + str(scale_now)) MATLAB_WORKSPACE = VmafConfig.root_path('matlab', 'SpEED') def _generate_result(self, asset): # routine to call the command-line executable and generate quality # scores in the log file. ref_workfile_path = asset.ref_workfile_path dis_workfile_path = asset.dis_workfile_path log_file_path = self._get_log_file_path(asset) current_dir = os.getcwd() + '/' ref_workfile_path = make_absolute_path(ref_workfile_path, current_dir) dis_workfile_path = make_absolute_path(dis_workfile_path, current_dir) log_file_path = make_absolute_path(log_file_path, current_dir) quality_width, quality_height = asset.quality_width_height speed_cmd = '''{matlab} -nodisplay -nosplash -nodesktop -r "run_speed('{ref}', '{dis}', {w}, {h}, {bands}, '{yuv_type}'); exit;" >> {log_file_path}'''.format( matlab=VmafExternalConfig.get_and_assert_matlab(), ref=ref_workfile_path, dis=dis_workfile_path, w=quality_width, h=quality_height, bands=self.scale_list, yuv_type=self._get_workfile_yuv_type(asset), log_file_path=log_file_path, ) if self.logger: self.logger.info(speed_cmd) os.chdir(self.MATLAB_WORKSPACE) run_process(speed_cmd, shell=True) os.chdir(current_dir) @classmethod def _post_process_result(cls, result): # override Executor._post_process_result def _speed(sspeed_tspeed): sspeed, tspeed = sspeed_tspeed if sspeed is not None and tspeed is not None: return sspeed * tspeed elif sspeed is None: return tspeed elif tspeed is None: return sspeed else: return None result = super(SpEEDMatlabFeatureExtractor, cls)._post_process_result(result) for scale_now in cls.scale_list: sspeed_scale_now_scores_key = cls.get_scores_key('sspeed_' + str(scale_now)) tspeed_scale_now_scores_key = cls.get_scores_key('tspeed_' + str(scale_now)) speed_scale_now_scores_key = cls.get_scores_key('speed_' + str(scale_now)) sspeed_scale_now_scores = result.result_dict[ sspeed_scale_now_scores_key] tspeed_scale_now_scores = result.result_dict[ tspeed_scale_now_scores_key] assert len(sspeed_scale_now_scores) == len(tspeed_scale_now_scores) # consistent with VMAF framework, which is to multiply S and T scores per frame, then average speed_scale_now_scores = map( _speed, zip(sspeed_scale_now_scores, tspeed_scale_now_scores)) result.result_dict[ speed_scale_now_scores_key] = speed_scale_now_scores # validate for feature in cls.DERIVED_ATOM_FEATURES: assert cls.get_scores_key(feature) in result.result_dict return result
def test_run_testing_psnr(self): exe = VmafConfig.root_path('run_testing') cmd = "{exe} PSNR {dataset} --parallelize --suppress-plot".format( exe=exe, dataset=self.dataset_filename) ret = run_process(cmd, shell=True) self.assertEquals(ret, 0)
from vmaf.config import VmafConfig from vmaf.core.executor import run_executors_in_parallel from vmaf.core.raw_extractor import DisYUVRawVideoExtractor from vmaf.core.nn_train_test_model import ToddNoiseClassifierTrainTestModel from vmaf.routine import read_dataset from vmaf.tools.misc import import_python_file # parameters num_train = 500 num_test = 50 n_epochs = 30 seed = 0 # None # read input dataset dataset_path = VmafConfig.resource_path('dataset', 'BSDS500_noisy_dataset.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) # shuffle assets np.random.seed(seed) np.random.shuffle(assets) assets = assets[:(num_train + num_test)] raw_video_h5py_filepath = VmafConfig.workdir_path('rawvideo.hdf5') raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file( raw_video_h5py_filepath) print '======================== Extract raw YUVs ==============================' _, raw_yuvs = run_executors_in_parallel(
def main(): if len(sys.argv) < 6: print_usage() return 2 try: fmt = sys.argv[1] width = int(sys.argv[2]) height = int(sys.argv[3]) ref_path = sys.argv[4] dis_path = sys.argv[5] except ValueError: print_usage() return 2 if width < 0 or height < 0: print("width and height must be non-negative, but are {w} and {h}".format(w=width, h=height)) print_usage() return 2 if fmt not in FMTS: print_usage() return 2 out_fmt = get_cmd_option(sys.argv, 6, len(sys.argv), '--out-fmt') if not (out_fmt is None or out_fmt == 'xml' or out_fmt == 'json' or out_fmt == 'text'): print_usage() return 2 pool_method = get_cmd_option(sys.argv, 6, len(sys.argv), '--pool') if not (pool_method is None or pool_method in POOL_METHODS): print('--pool can only have option among {}'.format(', '.join(POOL_METHODS))) return 2 asset = Asset(dataset="cmd", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={'width':width, 'height':height, 'yuv_type':fmt} ) assets = [asset] runner_class = PsnrQualityRunner runner = runner_class( assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, ) # run runner.run() result = runner.results[0] # pooling if pool_method == 'harmonic_mean': result.set_score_aggregate_method(ListStats.harmonic_mean) elif pool_method == 'min': result.set_score_aggregate_method(np.min) elif pool_method == 'median': result.set_score_aggregate_method(np.median) elif pool_method == 'perc5': result.set_score_aggregate_method(ListStats.perc5) elif pool_method == 'perc10': result.set_score_aggregate_method(ListStats.perc10) elif pool_method == 'perc20': result.set_score_aggregate_method(ListStats.perc20) else: # None or 'mean' pass # output if out_fmt == 'xml': print(result.to_xml()) elif out_fmt == 'json': print(result.to_json()) else: # None or 'text' print(str(result)) return 0
def test_run_vmaf_fextractor_with_resampling(self): ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324, 'quality_width': 160, 'quality_height': 90 }) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324, 'quality_width': 160, 'quality_height': 90 }) self.fextractor = VmafFeatureExtractor([asset, asset_original], None, fifo_mode=False) self.fextractor.run(parallelize=False) results = self.fextractor.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.74165043750000004, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 1.4066421666666666, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9807496875, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 27.319241250000001, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 1.4066421666666666, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.682829895833333, places=4)
def test_run_strred_fextractor_blackframes(self): ref_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv") dis_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_10.yuv") asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=VmafConfig.workdir_path(), ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) from vmaf.core.result_store import FileSystemResultStore result_store = FileSystemResultStore(logger=None) self.fextractor = StrredFeatureExtractor([asset, asset_original], None, fifo_mode=True, result_store=result_store) self.fextractor.run(parallelize=True) result0, result1 = self.fextractor.results import os self.assertTrue( os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue( os.path.exists(result_store._get_result_file_path(result1))) self.fextractor.run(parallelize=True) results = self.fextractor.results # ignore NaN for result in results: result.set_score_aggregate_method(ListStats.nonemean) self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 1220.5679849999999, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 50983.3097155, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 62228595.6081, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)