def test_run_speed_matlab_runner(self):
print 'test on running SPEED Matlab runner...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing(
)
self.runner = SpEEDMatlabQualityRunner([asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None)
self.runner.run(parallelize=True)
results = self.runner.results
self.assertAlmostEqual(
results[0]['SpEED_Matlab_feature_sspeed_4_score'],
5.155523354166667,
places=4)
self.assertAlmostEqual(
results[0]['SpEED_Matlab_feature_tspeed_4_score'],
15.091642416666668,
places=4)
self.assertAlmostEqual(results[0]['SpEED_Matlab_score'],
78.4927784076698,
places=4)
self.assertAlmostEqual(
results[1]['SpEED_Matlab_feature_sspeed_4_score'], 0.0, places=4)
self.assertAlmostEqual(
results[1]['SpEED_Matlab_feature_tspeed_4_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['SpEED_Matlab_score'], 0.0, places=4)
def test_run_noref_niqe_fextractor_train(self):
print 'test on running NIQE noref feature extractor in train mode...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = NiqeNorefFeatureExtractor(
[asset, asset_original],
None, fifo_mode=False,
result_store=None,
optional_dict={'mode': 'train'},
optional_dict2=None,
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha23_score'], 0.97259000000000073, places=4)
self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha13_score'], 0.80907000000000051, places=4)
self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha_m1_score'], 2.6135250000000019, places=4)
self.assertAlmostEqual(results[0]['NIQE_noref_feature_blbr1_score'], 0.9150526409258144, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha23_score'], 0.97447727272727347, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha13_score'], 0.89120909090909162, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha_m1_score'], 3.0300909090909118, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_blbr1_score'], 1.0508255408831713, places=4)
def test_feature_assembler_selected_atom_feature(self):
print 'test on feature assembler with selected atom features...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature':['vif', 'motion']},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_score']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_score']
def test_run_vmafossexec_runner_with_subsample2(self):
print 'test on running VMAFOSSEXEC runner with subsample2...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
subsample = 5
self.runner0 = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={}
)
self.runner0.run()
results0 = self.runner0.results
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={'subsample': subsample}
)
self.runner.run()
results = self.runner.results
for i in range(48):
if i % subsample == 0:
self.assertAlmostEqual(results0[0]['VMAFOSSEXEC_scores'][i], results[0]['VMAFOSSEXEC_scores'][i / subsample], places=7)
self.assertAlmostEqual(results0[1]['VMAFOSSEXEC_scores'][i], results[1]['VMAFOSSEXEC_scores'][i / subsample], places=7)
def test_run_noref_niqe_fextractor_with_patch_size(self):
print 'test on running NIQE noref feature extractor with custom patch size...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = NiqeNorefFeatureExtractor(
[asset, asset_original],
None, fifo_mode=False,
result_store=None,
optional_dict={'patch_size': 48},
optional_dict2=None,
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha23_score'], 0.8430156250000006, places=4)
self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha13_score'], 0.71714583333333359, places=4)
self.assertAlmostEqual(results[0]['NIQE_noref_feature_alpha_m1_score'], 2.2195590277777795, places=4)
self.assertAlmostEqual(results[0]['NIQE_noref_feature_blbr1_score'], 0.74061215376929412, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha23_score'], 0.9144918981481488, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha13_score'], 0.87132291666666728, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_alpha_m1_score'], 2.8193532986111136, places=4)
self.assertAlmostEqual(results[1]['NIQE_noref_feature_blbr1_score'], 0.99354006450609134, places=4)
def test_run_parallel_vmafossexec_runner_with_repeated_assets(self):
print 'test on running VMAFOSSEXEC quality runner in parallel with repeated assets...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing(
)
self.runner = VmafossExecQualityRunner(
[asset, asset_original, asset, asset],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None)
self.runner.run(parallelize=True)
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'],
76.699266666666674,
places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'],
99.946416666666664,
places=4)
self.assertAlmostEqual(results[2]['VMAFOSSEXEC_score'],
76.699266666666674,
places=3)
self.assertAlmostEqual(results[3]['VMAFOSSEXEC_score'],
76.699266666666674,
places=3)
def test_run_strrred_runner(self):
print 'test on running STRRED runner...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing(
)
self.runner = StrredQualityRunner([asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None)
self.runner.run(parallelize=True)
results = self.runner.results
self.assertEqual(self.runner.VERSION, "F1.2-1.1")
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'],
3.0114681041666671,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'],
7.3039486249999994,
places=4)
self.assertAlmostEqual(results[0]['STRRED_score'],
21.995608318659482,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_score'], 0.0, places=4)
def test_run_vmafossexec_runner_with_ci_and_phone_model(self):
print 'test on running VMAFOSSEXEC runner with conf interval and phone model...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.3", "vmaf_rb_v0.6.3.pkl"),
'phone_model':True,
'ci': True,
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 91.723012127641823, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 90.13159583333334, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 100.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 0.8371132083333332, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 0.0, places=4)
# per model score checks
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0001_score'], 90.25032499999999, places=3)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0002_score'], 88.18534583333333, places=3)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0003_score'], 89.04952291666666, places=3)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0020_score'], 90.16633958333334, places=3)
def test_run_vmafossexec_runner_with_motion2(self):
print 'test on running VMAFOSSEXEC runner with motion2 feature...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath':VmafConfig.test_resource_path("test_motion2.pkl")
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953522916666672, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953522916666672, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 78.532525000000007, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 97.089554166666673, places=4)
def test_run_parallel_moment_fextractor(self):
print 'test on running Moment feature extractor in parallel...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = MomentFeatureExtractor(
[asset, asset_original, asset],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['Moment_feature_ref1st_score'], 59.788567297525134, places=4)
self.assertAlmostEqual(results[0]['Moment_feature_ref2nd_score'], 4696.668388042269, places=4)
self.assertAlmostEqual(results[0]['Moment_feature_refvar_score'], 1121.519917231203, places=4)
self.assertAlmostEqual(results[0]['Moment_feature_dis1st_score'], 61.332006624999984, places=4)
self.assertAlmostEqual(results[0]['Moment_feature_dis2nd_score'], 4798.659574041666, places=4)
self.assertAlmostEqual(results[0]['Moment_feature_disvar_score'], 1036.837184348847, places=4)
self.assertAlmostEqual(results[1]['Moment_feature_ref1st_score'], 59.788567297525134, places=4)
self.assertAlmostEqual(results[1]['Moment_feature_ref2nd_score'], 4696.668388042269, places=4)
self.assertAlmostEqual(results[1]['Moment_feature_refvar_score'], 1121.519917231203, places=4)
self.assertAlmostEqual(results[1]['Moment_feature_dis1st_score'], 59.788567297525134, places=4)
self.assertAlmostEqual(results[1]['Moment_feature_dis2nd_score'], 4696.668388042269, places=4)
self.assertAlmostEqual(results[1]['Moment_feature_disvar_score'], 1121.519917231203, places=4)
self.assertAlmostEqual(results[2]['Moment_feature_ref1st_score'], 59.788567297525134, places=4)
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature':'all'},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.93458780728708746, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.509571520833333, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.271439270833337, places=4)
def test_noref_moment_fextractor_with_noref_asset(self):
print 'test on running Moment noref feature extractor on NorefAssets...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing(
)
self.fextractor = MomentNorefFeatureExtractor([asset, asset_original],
None,
fifo_mode=True,
result_store=None)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['Moment_noref_feature_1st_score'],
61.332006624999984)
self.assertAlmostEqual(results[0]['Moment_noref_feature_2nd_score'],
4798.659574041666)
self.assertAlmostEqual(results[0]['Moment_noref_feature_var_score'],
1036.8371843488285)
self.assertAlmostEqual(results[1]['Moment_noref_feature_1st_score'],
59.788567297525134)
self.assertAlmostEqual(results[1]['Moment_noref_feature_2nd_score'],
4696.668388042271)
self.assertAlmostEqual(results[1]['Moment_noref_feature_var_score'],
1121.519917231207)
def test_run_vmafossexec_runner_with_ci_and_phone_model(self):
print 'test on running VMAFOSSEXEC runner with conf interval and phone model...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.2", "vmaf_rb_v0.6.2.pkl"),
'phone_model':True,
'ci': True,
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 91.723012127641823, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 90.129761531349985, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 100.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 0.85880437658259945, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 0.0, places=4)
def test_from_xml_from_json_and_aggregation(self):
print 'test on running from_xml and from_json and aggregation...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing(
)
asset_list = [asset, asset_original]
self.runner = VmafQualityRunner(
asset_list,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath': VmafConfig.model_path("vmaf_v0.6.1.pkl"),
},
optional_dict2=None,
)
self.runner.run()
results = self.runner.results
xml_string_expected = results[0].to_xml()
xml_string_recon = Result.from_xml(xml_string_expected).to_xml()
json_string_expected = results[0].to_json()
json_string_recon = Result.from_json(json_string_expected).to_json()
assert xml_string_expected == xml_string_recon, "XML files do not match"
assert json_string_expected == json_string_recon, "JSON files do not match"
combined_result = Result.combine_result([results[0], results[1]])
# check that all keys are there
combined_result_keys = [key for key in combined_result.result_dict]
keys_0 = [key for key in results[0].result_dict]
keys_1 = [key for key in results[1].result_dict]
assert set(keys_0) == set(keys_1) == set(combined_result_keys)
# check that the dictionaries have been copied as expected
for key in combined_result_keys:
assert len(combined_result.result_dict[key]) == len(
results[0].result_dict[key]) + len(results[1].result_dict[key])
assert combined_result.result_dict[key][0] == results[
0].result_dict[key][0]
assert combined_result.result_dict[key][
len(results[0].result_dict[key]) -
1] == results[0].result_dict[key][
len(results[0].result_dict[key]) - 1]
assert combined_result.result_dict[key][len(
results[0].result_dict[key])] == results[1].result_dict[key][0]
assert combined_result.result_dict[key][
len(combined_result.result_dict[key]) -
1] == results[1].result_dict[key][
len(results[1].result_dict[key]) - 1]
def test_run_vmafossexec_runner_with_ci(self):
print 'test on running VMAFOSSEXEC runner with conf interval...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.3", "vmaf_rb_v0.6.3.pkl"),
'ci': True
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.93458777083333333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.443043750000001, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 73.10273541666668, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 99.79000416666668, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 1.1991330833333333, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 1.3028828125, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_low_score'], 70.82471875, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_low_score'], 94.79667083333334, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_high_score'], 74.85038125, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_high_score'], 99.99736666666666, places=4)
# per model score checks
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0001_score'], 73.26853333333334, places=3)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0002_score'], 70.38517916666667, places=3)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0003_score'], 71.59264583333334, places=3)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0020_score'], 73.15570625, places=3)
def test_run_psnr_fextractor(self):
print 'test on running PSNR feature extractor...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = PsnrFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['PSNR_feature_psnr_score'], 30.755063979166664, places=4)
self.assertAlmostEqual(results[1]['PSNR_feature_psnr_score'], 60.0, places=4)
def test_run_vmafossexec_runner_with_ci(self):
print 'test on running VMAFOSSEXEC runner with conf interval...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.2", "vmaf_rb_v0.6.2.pkl"),
'ci': True
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.93458777083333333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.443043750000001, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'], 73.099946626689174, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'], 99.686116179979152, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'], 1.2301198477788975, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'], 1.5917514683608882, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_low_score'], 70.801585803086553, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_low_score'], 94.784491176494996, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ci95_high_score'], 74.853442421187708, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ci95_high_score'], 99.992560767034618, places=4)
def test_run_vmafossexec_runner_with_subsample(self):
print 'test on running VMAFOSSEXEC runner with subsample...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={'subsample': 5}
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.954390000000018, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.742800000000003, places=4)
def test_run_ms_ssim_fextractor(self):
print 'test on running MS-SSIM feature extractor...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = MsSsimFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9981474583333334, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.96126793750000006, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.89773633333333336, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.99899612500000001, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9857694375, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.941185875, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.99923564583333324, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.997034020833, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.977992145833, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.99929210416666658, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.999588104167, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.99387125, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.99940356249999995, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.999907625, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.998222583333, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_vmafossexec_runner_with_thread(self):
print 'test on running VMAFOSSEXEC runner with thread...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={'thread': 3}
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.699271272486044, places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'],99.946416604585025, places=4)
def test_run_vmafossexec_runner_with_phone_score(self):
print 'test on running VMAFOSSEXEC runner with phone score...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'enable_transform_score': True,
}
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 92.542390144364546, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4)
def test_run_parallel_brisque_noref_fextractor(self):
print 'test on running BRISQUE noref feature extractor on NorefAssets in parallel...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractors, results = run_executors_in_parallel(
BrisqueNorefFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha23_score'], 0.78020833333333384, places=4)
self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha13_score'], 0.6322500000000002, places=4)
self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_N34_score'], -0.0071207420215536723, places=4)
self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha23_score'], 0.87156250000000046, places=4)
self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha13_score'], 0.82906250000000103, places=4)
self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_N34_score'], -0.0092448158862212092, places=4)
def test_run_strredOpt_fextractor(self):
print 'test on running STRREDopt feature extractor, no parallelization...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = StrredOptFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
# notice that these numbers are the same with ST-RRED, since the opt version should always produce identical results
self.assertAlmostEqual(results[0]['STRREDOpt_feature_srred_score'], 3.0166328541666663, places=4)
self.assertAlmostEqual(results[0]['STRREDOpt_feature_trred_score'], 7.338665770833333, places=4)
self.assertAlmostEqual(results[0]['STRREDOpt_feature_strred_score'], 22.336452104611016, places=4)
self.assertAlmostEqual(results[1]['STRREDOpt_feature_srred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRREDOpt_feature_trred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRREDOpt_feature_strred_score'], 0.0, places=4)
def test_run_strred_fextractor(self):
print 'test on running STRRED feature extractor, no parallelization...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = StrredFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0166328541666663, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.338665770833333, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 22.336452104611016, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_all_same_score'], 22.138060270044175, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_run_vmafossexec_runner_with_ci_and_custom_model(self):
print 'test on running VMAFOSSEXEC runner with conf interval and custom model...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing(
)
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath':
VmafConfig.test_resource_path('model',
'vmafplus_v0.5.2boot_test.pkl'),
'ci':
True
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'],
75.443043750000001,
places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'],
99.958047916666672,
places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_bagging_score'],
75.13012623785923,
places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_bagging_score'],
99.96504855577571,
places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_stddev_score'],
0.6812993325967104,
places=3)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_stddev_score'],
0.03947607207290399,
places=4)
def test_run_stmad_fextractor(self):
print 'test on running STMAD (Matlab) feature extractor, no parallelization...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = STMADFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
self.assertAlmostEqual(results[0].result_dict['STMAD_feature_smad_all_same_scores'][0], 2.889626, places=4)
self.assertAlmostEqual(results[0].result_dict['STMAD_feature_tmad_all_same_scores'][0], 5.649214, places=4)
self.assertAlmostEqual(results[0].result_dict['STMAD_feature_stmad_all_same_scores'][0], 4.983220, places=4)
self.assertAlmostEqual(results[1].result_dict['STMAD_feature_smad_all_same_scores'][0], 1.000000, places=4)
self.assertAlmostEqual(results[1].result_dict['STMAD_feature_tmad_all_same_scores'][0], 0.000000, places=4)
self.assertAlmostEqual(results[1].result_dict['STMAD_feature_stmad_all_same_scores'][0], -1.818097, places=4)
def test_run_SpEED_matlab_fextractor(self):
print 'test on running SpEED (Matlab) feature extractor, no paralellization...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = SpEEDMatlabFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
# S-SpEED assertions on first frame
self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_sspeed_2_scores'][0], 13.510418, places=4)
self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_sspeed_3_scores'][0], 7.211881, places=4)
self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_sspeed_4_scores'][0], 4.921501, places=4)
# T-SpEED assertions on third frame
self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_tspeed_2_scores'][2], 32.994605, places=4)
self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_tspeed_3_scores'][2], 22.404285, places=4)
self.assertAlmostEqual(results[0].result_dict[self.fextractor.TYPE + '_tspeed_4_scores'][2], 15.233468, places=4)
def test_run_noref_brisque_fextractor(self):
print 'test on running BRISQUE noref feature extractor...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.fextractor = BrisqueNorefFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha23_score'], 0.78020833333333384, places=4)
self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_alpha13_score'], 0.6322500000000002, places=4)
self.assertAlmostEqual(results[0]['BRISQUE_noref_feature_N34_score'], -0.0071207420215536723, places=4)
self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha23_score'], 0.87156250000000046, places=4)
self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_alpha13_score'], 0.82906250000000103, places=4)
self.assertAlmostEqual(results[1]['BRISQUE_noref_feature_N34_score'], -0.0092448158862212092, places=4)
def test_run_strrred_runner(self):
print 'test on running STRRED runner, with parallelization...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = StrredQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None
)
self.runner.run(parallelize=True)
results = self.runner.results
self.assertEqual(self.runner.VERSION, "F1.3-1.1")
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0166328541666663, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.338665770833333, places=4)
self.assertAlmostEqual(results[0]['STRRED_score'], 22.336452104611016, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_score'], 0.0, places=4)
def test_run_vmafossexec_runner_norm_type_none(self):
print 'test on running VMAFOSSEXEC runner with norm type none...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'model_filepath':VmafConfig.model_path("other_models", "nflxtrain_norm_type_none.pkl"),
},
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.93458777083333333, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 74.253349625150562, places=4)
self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 77.996338095161946, places=4)
