def test_run_speed_matlab_runner(self): print 'test on running SPEED Matlab runner...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing( ) self.runner = SpEEDMatlabQualityRunner([asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None) self.runner.run(parallelize=True) results = self.runner.results self.assertAlmostEqual( results[0]['SpEED_Matlab_feature_sspeed_4_score'], 5.155523354166667, places=4) self.assertAlmostEqual( results[0]['SpEED_Matlab_feature_tspeed_4_score'], 15.091642416666668, places=4) self.assertAlmostEqual(results[0]['SpEED_Matlab_score'], 78.4927784076698, places=4) self.assertAlmostEqual( results[1]['SpEED_Matlab_feature_sspeed_4_score'], 0.0, places=4) self.assertAlmostEqual( results[1]['SpEED_Matlab_feature_tspeed_4_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['SpEED_Matlab_score'], 0.0, places=4)
def test_run_noref_niqe_fextractor_train(self): print 'test on running NIQE noref feature extractor in train mode...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = NiqeNorefFeatureExtractor( [asset, asset_original], None, fifo_mode=False, result_store=None, optional_dict={'mode': 'train'}, optional_dict2=None, ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha23_score'], 0.97259000000000073, places=4) self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha13_score'], 0.80907000000000051, places=4) self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha_m1_score'], 2.6135250000000019, places=4) self.assertAlmostEqual(results[0]['NIQE_noref_feature_blbr1_score'], 0.9150526409258144, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha23_score'], 0.97447727272727347, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha13_score'], 0.89120909090909162, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha_m1_score'], 3.0300909090909118, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_blbr1_score'], 1.0508255408831713, places=4)
def test_feature_assembler_selected_atom_feature(self): print 'test on feature assembler with selected atom features...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fassembler = FeatureAssembler( feature_dict={'VMAF_feature':['vif', 'motion']}, feature_option_dict=None, assets=[asset, asset_original], logger=None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, parallelize=True, ) self.fassembler.run() results = self.fassembler.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4) with self.assertRaises(KeyError): results[0]['VMAF_feature_ansnr_scores'] with self.assertRaises(KeyError): results[0]['VMAF_feature_ansnr_score'] with self.assertRaises(KeyError): results[0]['VMAF_feature_adm_scores'] with self.assertRaises(KeyError): results[0]['VMAF_feature_adm_score']
def test_run_vmafossexec_runner_with_subsample2(self): print 'test on running VMAFOSSEXEC runner with subsample2...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() subsample = 5 self.runner0 = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={} ) self.runner0.run() results0 = self.runner0.results self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={'subsample': subsample} ) self.runner.run() results = self.runner.results for i in range(48): if i % subsample == 0: self.assertAlmostEqual(results0[0]['VMAFOSSEXEC_scores'][i], results[0]['VMAFOSSEXEC_scores'][i / subsample], places=7) self.assertAlmostEqual(results0[1]['VMAFOSSEXEC_scores'][i], results[1]['VMAFOSSEXEC_scores'][i / subsample], places=7)
def test_run_noref_niqe_fextractor_with_patch_size(self): print 'test on running NIQE noref feature extractor with custom patch size...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = NiqeNorefFeatureExtractor( [asset, asset_original], None, fifo_mode=False, result_store=None, optional_dict={'patch_size': 48}, optional_dict2=None, ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha23_score'], 0.8430156250000006, places=4) self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha13_score'], 0.71714583333333359, places=4) self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha_m1_score'], 2.2195590277777795, places=4) self.assertAlmostEqual(results[0]['NIQE_noref_feature_blbr1_score'], 0.74061215376929412, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha23_score'], 0.9144918981481488, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha13_score'], 0.87132291666666728, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha_m1_score'], 2.8193532986111136, places=4) self.assertAlmostEqual(results[1]['NIQE_noref_feature_blbr1_score'], 0.99354006450609134, places=4)
def test_run_parallel_vmafossexec_runner_with_repeated_assets(self): print 'test on running VMAFOSSEXEC quality runner in parallel with repeated assets...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing( ) self.runner = VmafossExecQualityRunner( [asset, asset_original, asset, asset], None, fifo_mode=True, delete_workdir=True, result_store=None) self.runner.run(parallelize=True) results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.699266666666674, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.946416666666664, places=4) self.assertAlmostEqual(results[2]['VMAFOSSEXEC_score'], 76.699266666666674, places=3) self.assertAlmostEqual(results[3]['VMAFOSSEXEC_score'], 76.699266666666674, places=3)
def test_run_strrred_runner(self): print 'test on running STRRED runner...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing( ) self.runner = StrredQualityRunner([asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None) self.runner.run(parallelize=True) results = self.runner.results self.assertEqual(self.runner.VERSION, "F1.2-1.1") self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0114681041666671, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.3039486249999994, places=4) self.assertAlmostEqual(results[0]['STRRED_score'], 21.995608318659482, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_score'], 0.0, places=4)
def test_run_vmafossexec_runner_with_ci_and_phone_model(self): print 'test on running VMAFOSSEXEC runner with conf interval and phone model...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.3", "vmaf_rb_v0.6.3.pkl"), 'phone_model':True, 'ci': True, }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 91.723012127641823, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 90.13159583333334, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 100.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 0.8371132083333332, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 0.0, places=4) # per model score checks self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0001_score'], 90.25032499999999, places=3) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0002_score'], 88.18534583333333, places=3) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0003_score'], 89.04952291666666, places=3) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0020_score'], 90.16633958333334, places=3)
def test_run_vmafossexec_runner_with_motion2(self): print 'test on running VMAFOSSEXEC runner with motion2 feature...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath':VmafConfig.test_resource_path("test_motion2.pkl") }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953522916666672, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953522916666672, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 78.532525000000007, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 97.089554166666673, places=4)
def test_run_parallel_moment_fextractor(self): print 'test on running Moment feature extractor in parallel...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = MomentFeatureExtractor( [asset, asset_original, asset], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=True) results = self.fextractor.results self.assertAlmostEqual(results[0]['Moment_feature_ref1st_score'], 59.788567297525134, places=4) self.assertAlmostEqual(results[0]['Moment_feature_ref2nd_score'], 4696.668388042269, places=4) self.assertAlmostEqual(results[0]['Moment_feature_refvar_score'], 1121.519917231203, places=4) self.assertAlmostEqual(results[0]['Moment_feature_dis1st_score'], 61.332006624999984, places=4) self.assertAlmostEqual(results[0]['Moment_feature_dis2nd_score'], 4798.659574041666, places=4) self.assertAlmostEqual(results[0]['Moment_feature_disvar_score'], 1036.837184348847, places=4) self.assertAlmostEqual(results[1]['Moment_feature_ref1st_score'], 59.788567297525134, places=4) self.assertAlmostEqual(results[1]['Moment_feature_ref2nd_score'], 4696.668388042269, places=4) self.assertAlmostEqual(results[1]['Moment_feature_refvar_score'], 1121.519917231203, places=4) self.assertAlmostEqual(results[1]['Moment_feature_dis1st_score'], 59.788567297525134, places=4) self.assertAlmostEqual(results[1]['Moment_feature_dis2nd_score'], 4696.668388042269, places=4) self.assertAlmostEqual(results[1]['Moment_feature_disvar_score'], 1121.519917231203, places=4) self.assertAlmostEqual(results[2]['Moment_feature_ref1st_score'], 59.788567297525134, places=4)
def test_feature_assembler_whole_feature(self): print 'test on feature assembler with whole feature...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fassembler = FeatureAssembler( feature_dict={'VMAF_feature':'all'}, feature_option_dict=None, assets=[asset, asset_original], logger=None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=None, optional_dict2=None, parallelize=True, ) self.fassembler.run() results = self.fassembler.results self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.93458780728708746, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.509571520833333, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.271439270833337, places=4)
def test_noref_moment_fextractor_with_noref_asset(self): print 'test on running Moment noref feature extractor on NorefAssets...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing( ) self.fextractor = MomentNorefFeatureExtractor([asset, asset_original], None, fifo_mode=True, result_store=None) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['Moment_noref_feature_1st_score'], 61.332006624999984) self.assertAlmostEqual(results[0]['Moment_noref_feature_2nd_score'], 4798.659574041666) self.assertAlmostEqual(results[0]['Moment_noref_feature_var_score'], 1036.8371843488285) self.assertAlmostEqual(results[1]['Moment_noref_feature_1st_score'], 59.788567297525134) self.assertAlmostEqual(results[1]['Moment_noref_feature_2nd_score'], 4696.668388042271) self.assertAlmostEqual(results[1]['Moment_noref_feature_var_score'], 1121.519917231207)
def test_run_vmafossexec_runner_with_ci_and_phone_model(self): print 'test on running VMAFOSSEXEC runner with conf interval and phone model...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.2", "vmaf_rb_v0.6.2.pkl"), 'phone_model':True, 'ci': True, }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 91.723012127641823, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 90.129761531349985, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 100.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 0.85880437658259945, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 0.0, places=4)
def test_from_xml_from_json_and_aggregation(self): print 'test on running from_xml and from_json and aggregation...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing( ) asset_list = [asset, asset_original] self.runner = VmafQualityRunner( asset_list, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath': VmafConfig.model_path("vmaf_v0.6.1.pkl"), }, optional_dict2=None, ) self.runner.run() results = self.runner.results xml_string_expected = results[0].to_xml() xml_string_recon = Result.from_xml(xml_string_expected).to_xml() json_string_expected = results[0].to_json() json_string_recon = Result.from_json(json_string_expected).to_json() assert xml_string_expected == xml_string_recon, "XML files do not match" assert json_string_expected == json_string_recon, "JSON files do not match" combined_result = Result.combine_result([results[0], results[1]]) # check that all keys are there combined_result_keys = [key for key in combined_result.result_dict] keys_0 = [key for key in results[0].result_dict] keys_1 = [key for key in results[1].result_dict] assert set(keys_0) == set(keys_1) == set(combined_result_keys) # check that the dictionaries have been copied as expected for key in combined_result_keys: assert len(combined_result.result_dict[key]) == len( results[0].result_dict[key]) + len(results[1].result_dict[key]) assert combined_result.result_dict[key][0] == results[ 0].result_dict[key][0] assert combined_result.result_dict[key][ len(results[0].result_dict[key]) - 1] == results[0].result_dict[key][ len(results[0].result_dict[key]) - 1] assert combined_result.result_dict[key][len( results[0].result_dict[key])] == results[1].result_dict[key][0] assert combined_result.result_dict[key][ len(combined_result.result_dict[key]) - 1] == results[1].result_dict[key][ len(results[1].result_dict[key]) - 1]
def test_run_vmafossexec_runner_with_ci(self): print 'test on running VMAFOSSEXEC runner with conf interval...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.3", "vmaf_rb_v0.6.3.pkl"), 'ci': True }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.93458777083333333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.443043750000001, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 73.10273541666668, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 99.79000416666668, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 1.1991330833333333, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 1.3028828125, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_low_score'], 70.82471875, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_low_score'], 94.79667083333334, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_high_score'], 74.85038125, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_high_score'], 99.99736666666666, places=4) # per model score checks self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0001_score'], 73.26853333333334, places=3) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0002_score'], 70.38517916666667, places=3) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0003_score'], 71.59264583333334, places=3) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0020_score'], 73.15570625, places=3)
def test_run_psnr_fextractor(self): print 'test on running PSNR feature extractor...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = PsnrFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['PSNR_feature_psnr_score'], 30.755063979166664, places=4) self.assertAlmostEqual(results[1]['PSNR_feature_psnr_score'], 60.0, places=4)
def test_run_vmafossexec_runner_with_ci(self): print 'test on running VMAFOSSEXEC runner with conf interval...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.2", "vmaf_rb_v0.6.2.pkl"), 'ci': True }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.93458777083333333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.443043750000001, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 73.099946626689174, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 99.686116179979152, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 1.2301198477788975, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 1.5917514683608882, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_low_score'], 70.801585803086553, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_low_score'], 94.784491176494996, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_high_score'], 74.853442421187708, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_high_score'], 99.992560767034618, places=4)
def test_run_vmafossexec_runner_with_subsample(self): print 'test on running VMAFOSSEXEC runner with subsample...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={'subsample': 5} ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.954390000000018, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.742800000000003, places=4)
def test_run_ms_ssim_fextractor(self): print 'test on running MS-SSIM feature extractor...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = MsSsimFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9981474583333334, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.96126793750000006, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.89773633333333336, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.99899612500000001, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9857694375, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.941185875, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.99923564583333324, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.997034020833, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.977992145833, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.99929210416666658, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.999588104167, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.99387125, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.99940356249999995, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.999907625, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.998222583333, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_vmafossexec_runner_with_thread(self): print 'test on running VMAFOSSEXEC runner with thread...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={'thread': 3} ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.699271272486044, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'],99.946416604585025, places=4)
def test_run_vmafossexec_runner_with_phone_score(self): print 'test on running VMAFOSSEXEC runner with phone score...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'enable_transform_score': True, } ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 92.542390144364546, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4)
def test_run_parallel_brisque_noref_fextractor(self): print 'test on running BRISQUE noref feature extractor on NorefAssets in parallel...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractors, results = run_executors_in_parallel( BrisqueNorefFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha23_score'], 0.78020833333333384, places=4) self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha13_score'], 0.6322500000000002, places=4) self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_N34_score'], -0.0071207420215536723, places=4) self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha23_score'], 0.87156250000000046, places=4) self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha13_score'], 0.82906250000000103, places=4) self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_N34_score'], -0.0092448158862212092, places=4)
def test_run_strredOpt_fextractor(self): print 'test on running STRREDopt feature extractor, no parallelization...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = StrredOptFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=False) results = self.fextractor.results # notice that these numbers are the same with ST-RRED, since the opt version should always produce identical results self.assertAlmostEqual(results[0]['STRREDOpt_feature_srred_score'], 3.0166328541666663, places=4) self.assertAlmostEqual(results[0]['STRREDOpt_feature_trred_score'], 7.338665770833333, places=4) self.assertAlmostEqual(results[0]['STRREDOpt_feature_strred_score'], 22.336452104611016, places=4) self.assertAlmostEqual(results[1]['STRREDOpt_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRREDOpt_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRREDOpt_feature_strred_score'], 0.0, places=4)
def test_run_strred_fextractor(self): print 'test on running STRRED feature extractor, no parallelization...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = StrredFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=False) results = self.fextractor.results self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0166328541666663, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.338665770833333, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 22.336452104611016, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_strred_all_same_score'], 22.138060270044175, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_run_vmafossexec_runner_with_ci_and_custom_model(self): print 'test on running VMAFOSSEXEC runner with conf interval and custom model...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing( ) self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath': VmafConfig.test_resource_path('model', 'vmafplus_v0.5.2boot_test.pkl'), 'ci': True }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.443043750000001, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 75.13012623785923, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 99.96504855577571, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 0.6812993325967104, places=3) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 0.03947607207290399, places=4)
def test_run_stmad_fextractor(self): print 'test on running STMAD (Matlab) feature extractor, no parallelization...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = STMADFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=False) results = self.fextractor.results self.assertAlmostEqual(results[0].result_dict['STMAD_feature_smad_all_same_scores'][0], 2.889626, places=4) self.assertAlmostEqual(results[0].result_dict['STMAD_feature_tmad_all_same_scores'][0], 5.649214, places=4) self.assertAlmostEqual(results[0].result_dict['STMAD_feature_stmad_all_same_scores'][0], 4.983220, places=4) self.assertAlmostEqual(results[1].result_dict['STMAD_feature_smad_all_same_scores'][0], 1.000000, places=4) self.assertAlmostEqual(results[1].result_dict['STMAD_feature_tmad_all_same_scores'][0], 0.000000, places=4) self.assertAlmostEqual(results[1].result_dict['STMAD_feature_stmad_all_same_scores'][0], -1.818097, places=4)
def test_run_SpEED_matlab_fextractor(self): print 'test on running SpEED (Matlab) feature extractor, no paralellization...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = SpEEDMatlabFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run(parallelize=False) results = self.fextractor.results # S-SpEED assertions on first frame self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_sspeed_2_scores'][0], 13.510418, places=4) self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_sspeed_3_scores'][0], 7.211881, places=4) self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_sspeed_4_scores'][0], 4.921501, places=4) # T-SpEED assertions on third frame self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_tspeed_2_scores'][2], 32.994605, places=4) self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_tspeed_3_scores'][2], 22.404285, places=4) self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_tspeed_4_scores'][2], 15.233468, places=4)
def test_run_noref_brisque_fextractor(self): print 'test on running BRISQUE noref feature extractor...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.fextractor = BrisqueNorefFeatureExtractor( [asset, asset_original], None, fifo_mode=True, result_store=None ) self.fextractor.run() results = self.fextractor.results self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha23_score'], 0.78020833333333384, places=4) self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha13_score'], 0.6322500000000002, places=4) self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_N34_score'], -0.0071207420215536723, places=4) self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha23_score'], 0.87156250000000046, places=4) self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha13_score'], 0.82906250000000103, places=4) self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_N34_score'], -0.0092448158862212092, places=4)
def test_run_strrred_runner(self): print 'test on running STRRED runner, with parallelization...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = StrredQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None ) self.runner.run(parallelize=True) results = self.runner.results self.assertEqual(self.runner.VERSION, "F1.3-1.1") self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0166328541666663, places=4) self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.338665770833333, places=4) self.assertAlmostEqual(results[0]['STRRED_score'], 22.336452104611016, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4) self.assertAlmostEqual(results[1]['STRRED_score'], 0.0, places=4)
def test_run_vmafossexec_runner_norm_type_none(self): print 'test on running VMAFOSSEXEC runner with norm type none...' ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() self.runner = VmafossExecQualityRunner( [asset, asset_original], None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict={ 'model_filepath':VmafConfig.model_path("other_models", "nflxtrain_norm_type_none.pkl"), }, ) self.runner.run() results = self.runner.results self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.93458777083333333, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 74.253349625150562, places=4) self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 77.996338095161946, places=4)