def test_run_vmaf_fextractor_with_resampling(self):
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324,
'quality_width':160, 'quality_height':90})
asset_original = Asset(dataset="test", content_id=0, asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324,
'quality_width':160, 'quality_height':90})
self.fextractor = VmafFeatureExtractor(
[asset, asset_original], None, fifo_mode=False)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.782546520833, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],1.3216766875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.98229347916666665, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 28.0085990417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 1.3216766875, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.811350125, places=4)
def test_run_strred_fextractor(self):
print 'test on running STRRED feature extractor...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractor = StrredFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 3.0114681041666671, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 7.3039486249999994, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 21.995608318659482, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_checkerboard_identical(self):
print 'test on checkerboard pattern identical...'
LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_checkerboardIdentical'
ref_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
yuv_fmt = "yuv420p"
yuv_width = 1920
yuv_height = 1080
cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv,
dis=dis_yuv, w=yuv_width, h=yuv_height,
log=LOCAL_LOG_FILENAME)
run_process(cmd, shell=True)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 1.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], 21.1138813333, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 12.554836666666667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 1.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 2773.891225, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 2773.891225, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 33021350.5, places=-3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 33021387.0625, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 29.8567246667, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0], 25757432.0, places=-3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 25757473.3333, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 259774.958333, places=1)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 259774.9375, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 277.120382, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 277.120382, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 924.193766, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 924.193766, places=3)
def test_run_asset_extractor(self):
print 'test on running asset extractor...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324,
'quality_width':160, 'quality_height':90})
asset_original = Asset(dataset="test", content_id=0, asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324,
'quality_width':160, 'quality_height':90})
self.fextractor = AssetExtractor(
[asset, asset_original], None, fifo_mode=True)
self.fextractor.run()
results = self.fextractor.results
self.assertEqual(str(results[0]['asset']), 'test_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_160x90')
self.assertEqual(str(results[1]['asset']), 'test_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc00_576x324_576x324_q_160x90')
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertEqual(str(results[0]['asset']), 'test_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_160x90')
self.assertEqual(str(results[1]['asset']), 'test_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc00_576x324_576x324_q_160x90')
def test_checkerboard_shifted_by_1(self):
print 'test on checkerboard pattern shifted by 1...'
LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_checkerboard_shifted_by_1'
ref_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
yuv_fmt = "yuv420p"
yuv_width = 1920
yuv_height = 1080
cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv,
dis=dis_yuv, w=yuv_width, h=yuv_height,
log=LOCAL_LOG_FILENAME)
run_process(cmd, shell=True)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 0.7853383333333334, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], 7.92623066667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 12.5548366667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 0.156834666667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 2178.5352886666665, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 2773.891225, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 5178894.51562, places=-1)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 33021387.0625, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 16.669074, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0], 2908829.0, places=-1)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 25757473.3333, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 128957.796875, places=-2)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 259774.9375, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 201.15329999999997, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 277.120382, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 765.1562903333333, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 924.193766, places=3)
def test_run_psnr_fextractor(self):
print 'test on running PSNR feature extractor...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractor = PsnrFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['PSNR_feature_psnr_score'], 30.755063979166664, places=4)
self.assertAlmostEqual(results[1]['PSNR_feature_psnr_score'], 60.0, places=4)
def test_explain_vmaf_results(self):
print 'test on running VMAF runner with local explainer...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.runner = VmafQualityRunnerWithLocalExplainer(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict2={'explainer': LocalExplainer(neighbor_samples=100)}
)
np.random.seed(0)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAF_score'], 76.699271272486044, places=4)
self.assertAlmostEqual(results[1]['VMAF_score'], 99.946416604585025, places=4)
expected_feature_names = ['VMAF_feature_adm2_score',
'VMAF_feature_motion2_score',
'VMAF_feature_vif_scale0_score',
'VMAF_feature_vif_scale1_score',
'VMAF_feature_vif_scale2_score',
'VMAF_feature_vif_scale3_score']
weights = np.mean(results[0]['VMAF_scores_exps']['feature_weights'], axis=0)
self.assertAlmostEqual(weights[0], 0.66021689480916868, places=4)
self.assertAlmostEqual(weights[1], 0.14691682562211777, places=4)
self.assertAlmostEqual(weights[2], -0.023682744847036086, places=4)
self.assertAlmostEqual(weights[3], -0.029779341850172818, places=4)
self.assertAlmostEqual(weights[4], 0.19149485210137338, places=4)
self.assertAlmostEqual(weights[5], 0.31890978778344126, places=4)
self.assertEqual(results[0]['VMAF_scores_exps']['feature_names'],
expected_feature_names)
weights = np.mean(results[1]['VMAF_scores_exps']['feature_weights'], axis=0)
self.assertAlmostEqual(weights[0], 0.69597961598838509, places=4)
self.assertAlmostEqual(weights[1], 0.18256016705513464, places=4)
self.assertAlmostEqual(weights[2], 0.0090048099912423147, places=4)
self.assertAlmostEqual(weights[3], 0.028671810808880094, places=4)
self.assertAlmostEqual(weights[4], 0.21935602577417926, places=4)
self.assertAlmostEqual(weights[5], 0.34190431429767715, places=4)
self.assertEqual(results[1]['VMAF_scores_exps']['feature_names'],
expected_feature_names)
def test_flat_value10(self):
print 'test on flat pattern of value 10...'
LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_flat_value10'
ref_yuv = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv")
dis_yuv = VmafConfig.test_resource_path("yuv", "flat_1920_1080_10.yuv")
yuv_fmt = "yuv420p"
yuv_width = 1920
yuv_height = 1080
cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv,
dis=dis_yuv, w=yuv_width, h=yuv_height,
log=LOCAL_LOG_FILENAME)
run_process(cmd, shell=True)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 1.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], 21.899511, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 0.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 1.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 149.780313, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 149.780392, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 2753999.99219, places=1)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 2754000.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 29.045954, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0],2073600.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 2073600.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 32400.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 32400.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 65.573967, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 65.573967, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 16.667711, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 16.667747, places=3)
def test_checkerboard_opposite(self):
print 'test on checkerboard pattern opposite...'
LOCAL_LOG_FILENAME = self.LOG_FILENAME + '_checkerboard_opposite'
ref_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_yuv = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv")
yuv_fmt = "yuv420p"
yuv_width = 1920
yuv_height = 1080
cmd = self.CMD_TEMPLATE.format(vmaf=required(ExternalProgram.vmaf), fmt=yuv_fmt, ref=ref_yuv,
dis=dis_yuv, w=yuv_width, h=yuv_height,
log=LOCAL_LOG_FILENAME)
run_process(cmd, shell=True)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm")[0], 0.053996333333333334, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "ansnr")[0], -5.758091333333334, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "motion")[0], 12.554836666666667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif")[0], 0.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num")[0], 149.780313, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den")[0], 2773.891225, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num")[0], 6.66666666667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den")[0], 33021387.0625, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "anpsnr")[0], 2.984752, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale0")[0], 6.66666666667, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale0")[0], 25757473.3333, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_num_scale3")[0], 0.0, places=4)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "vif_den_scale3")[0], 259774.9375, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale0")[0], 65.573967, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale0")[0], 277.120382, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_num_scale3")[0], 16.667711, places=3)
self.assertAlmostEquals(read_log(LOCAL_LOG_FILENAME, "adm_den_scale3")[0], 924.193766, places=3)
def test_run_dis_yuv_raw_video_extractor_parallel(self):
print 'test on running dis YUV raw video extractor...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath)
self.fextractor = DisYUVRawVideoExtractor(
[asset, asset_original], None, fifo_mode=False,
optional_dict={'channels': 'yu'},
optional_dict2={'h5py_file': h5py_file}
)
with self.assertRaises(AssertionError):
self.fextractor.run(parallelize=True)
DisYUVRawVideoExtractor.close_h5py_file(h5py_file)
def test_train_test_on_raw_dataset_with_dis1st_thr(self):
train_dataset = import_python_file(
VmafConfig.test_resource_path('raw_dataset_sample.py'))
model_param = import_python_file(
VmafConfig.test_resource_path('model_param_sample.py'))
feature_param = import_python_file(
VmafConfig.test_resource_path('feature_param_sample.py'))
train_fassembler, train_assets, train_stats, test_fassembler, test_assets, test_stats, _ = train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=train_dataset,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=None,
parallelize=True,
logger=None,
fifo_mode=True,
output_model_filepath=self.output_model_filepath
)
self.train_fassembler = train_fassembler
self.assertTrue(os.path.exists(self.output_model_filepath))
self.assertAlmostEqual(train_stats['ys_label_pred'][0], 93.565459224020742, places=3)
self.assertAlmostEqual(test_stats['ys_label_pred'][0], 93.565459224020742, places=3)
def test_run_parallel_moment_noref_fextractor(self):
print 'test on running Moment noref feature extractor on NorefAssets in parallel...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = NorefAsset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = NorefAsset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
fextractor = MomentNorefFeatureExtractor(
[asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
)
self.fextractors = [fextractor]
fextractor.run(parallelize=True)
results = fextractor.results
self.assertAlmostEqual(results[0]['Moment_noref_feature_1st_score'], 61.332006624999984)
self.assertAlmostEqual(results[0]['Moment_noref_feature_2nd_score'], 4798.659574041666)
self.assertAlmostEqual(results[0]['Moment_noref_feature_var_score'], 1036.8371843488285)
self.assertAlmostEqual(results[1]['Moment_noref_feature_1st_score'], 59.788567297525134)
self.assertAlmostEqual(results[1]['Moment_noref_feature_2nd_score'], 4696.668388042271)
self.assertAlmostEqual(results[1]['Moment_noref_feature_var_score'], 1121.519917231207)
def test_run_parallel_ssim_fextractor(self):
print 'test on running SSIM feature extractor in parallel...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractor = SsimFeatureExtractor(
[asset, asset_original, asset],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_score'], 0.86322654166666657, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_l_score'], 0.9981474583333334, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_c_score'], 0.96126793750000006, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_s_score'], 0.89773633333333336, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_l_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_c_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_s_score'], 1.0, places=4)
self.assertAlmostEqual(results[2]['SSIM_feature_ssim_score'], 0.86322654166666657, places=4)
def test_run_psnr(self):
exe = VmafConfig.root_path('run_psnr')
line = 'yuv420p 576 324 {root}/python/test/resource/yuv/src01_hrc00_576x324.yuv ' \
'{root}/python/test/resource/yuv/src01_hrc01_576x324.yuv'.format(root=VmafConfig.root_path())
cmd = "{exe} {line} >/dev/null 2>&1".format(line=line, exe=exe)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
def test_run_ms_ssim_fextractor(self):
print 'test on running MS-SSIM feature extractor...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractor = MsSsimFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=None
)
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9981474583333334, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.96126793750000006, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.89773633333333336, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.99899612500000001, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9857694375, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.941185875, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.99923564583333324, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.997034020833, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.977992145833, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.99929210416666658, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.999588104167, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.99387125, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.99940356249999995, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.999907625, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.998222583333, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_cleaning_cache_psnr(self):
exe = VmafConfig.root_path('run_testing')
cmd = "{exe} PSNR {dataset} --parallelize --cache-result --suppress-plot".format(
exe=exe, dataset=self.dataset_filename)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
exe = VmafConfig.root_path('python', 'script', 'run_cleaning_cache.py')
cmd = "{exe} PSNR {dataset}".format(
exe=exe, dataset=self.dataset_filename)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
def test_run_vmaf_in_batch(self):
line = 'yuv420p 576 324 {root}/python/test/resource/yuv/src01_hrc00_576x324.yuv ' \
'{root}/python/test/resource/yuv/src01_hrc01_576x324.yuv'.format(root=VmafConfig.root_path())
cmd = 'echo "{line}" > {batch_filename}'.format(
line=line, batch_filename=self.batch_filename)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
exe = VmafConfig.root_path('run_vmaf_in_batch')
cmd = "{exe} {input} --parallelize >/dev/null 2>&1".format(
exe=exe, input=self.batch_filename)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
def test_bitrate(self):
ref_path = VmafConfig.test_resource_path('yuv/', 'src01_hrc00_576x324.yuv')
dis_path = VmafConfig.test_resource_path('yuv', 'src01_hrc01_576x324.yuv')
asset = Asset(dataset="test", content_id=0, asset_id=0,
ref_path=ref_path, dis_path=dis_path,
asset_dict={'ref_start_frame':0, 'ref_end_frame':47,
'dis_start_frame':0, 'dis_end_frame':47,
'fps':23.976},)
self.assertAlmostEquals(asset.ref_bitrate_kbps_for_entire_file,
53693.964287999996, places=4)
self.assertAlmostEquals(asset.dis_bitrate_kbps_for_entire_file,
53693.964287999996, places=4)
def setUp(self):
ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':1920, 'height':1080})
self.runner = VmafLegacyQualityRunner(
[asset], None, fifo_mode=True,
delete_workdir=True, result_store=None,
)
self.runner.run()
self.result = self.runner.results[0]
def test_run_testlib_runner(self):
print 'test on running TESTLIB runner...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.runner = TestLibRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=None,
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['TESTLIB_vif_scale0_score'],0.363420458333, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_vif_scale1_score'], 0.766647520833, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_vif_scale2_score'], 0.862854708333, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_vif_scale3_score'], 0.915971791667, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_adm2_score'], 0.93458777083333333, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_psnr_score'], 30.7550666667, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_ssim_score'], 0.86322654166666657, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_vif_scale1_score'],0.999999958333, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_vif_scale2_score'],0.999999416667, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_vif_scale3_score'], 0.999999208333, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_psnr_score'], 60.0, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['TESTLIB_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[0]['TESTLIB_score'], 76.699271272486044, places=3)
self.assertAlmostEqual(results[1]['TESTLIB_score'],99.946416604585025, places=4)
def run_vmaf_cv_from_raw(train_dataset_raw_filepath, test_dataset_raw_filepath,
param_filepath, output_model_filepath, **kwargs):
if 'train_quality_wh' in kwargs:
train_quality_width, train_quality_height = kwargs['train_quality_wh']
else:
train_quality_width = None
train_quality_height = None
if 'test_quality_wh' in kwargs:
test_quality_width, test_quality_height = kwargs['test_quality_wh']
else:
test_quality_width = None
test_quality_height = None
if 'train_transform_final' in kwargs:
train_transform_final = kwargs['train_transform_final']
else:
train_transform_final = None
if 'test_transform_final' in kwargs:
test_transform_final = kwargs['test_transform_final']
else:
test_transform_final = None
train_output_dataset_filepath = VmafConfig.workspace_path('dataset', 'train_dataset.py')
generate_dataset_from_raw(raw_dataset_filepath=train_dataset_raw_filepath,
output_dataset_filepath=train_output_dataset_filepath,
quality_width=train_quality_width,
quality_height=train_quality_height,
transform_final=train_transform_final,
**kwargs)
test_output_dataset_filepath = VmafConfig.workspace_path('dataset', 'test_dataset.py') \
if test_dataset_raw_filepath is not None else None
generate_dataset_from_raw(raw_dataset_filepath=test_dataset_raw_filepath,
output_dataset_filepath=test_output_dataset_filepath,
quality_width=test_quality_width,
quality_height=test_quality_height,
transform_final=test_transform_final,
**kwargs)
run_vmaf_cv(
train_dataset_filepath=train_output_dataset_filepath,
test_dataset_filepath=test_output_dataset_filepath,
param_filepath=param_filepath,
output_model_filepath=output_model_filepath,
**kwargs
)
def test_read_dataset_mixed(self):
dataset_path = VmafConfig.test_resource_path('test_dataset_mixed.py')
dataset = import_python_file(dataset_path)
assets = read_dataset(dataset)
self.assertEquals(len(assets), 4)
self.assertEqual(assets[0].resampling_type, 'bilinear')
self.assertEqual(assets[0].ref_yuv_type, 'yuv420p')
self.assertEqual(assets[0].dis_yuv_type, 'yuv420p')
self.assertEqual(assets[0].ref_width_height, (1920, 1080))
self.assertEqual(assets[0].dis_width_height, (1920, 1080))
self.assertEqual(assets[1].resampling_type, 'bilinear')
self.assertEqual(assets[1].ref_yuv_type, 'yuv420p')
self.assertEqual(assets[1].dis_yuv_type, 'notyuv')
self.assertEqual(assets[1].ref_width_height, (1920, 1080))
self.assertEqual(assets[1].dis_width_height, None)
self.assertEqual(assets[2].resampling_type, 'bilinear')
self.assertEqual(assets[2].ref_yuv_type, 'yuv420p')
self.assertEqual(assets[2].dis_yuv_type, 'yuv420p')
self.assertEqual(assets[2].ref_width_height, (720, 480))
self.assertEqual(assets[2].dis_width_height, (720, 480))
self.assertEqual(assets[3].resampling_type, 'bilinear')
self.assertEqual(assets[3].ref_yuv_type, 'yuv420p')
self.assertEqual(assets[3].dis_yuv_type, 'notyuv')
self.assertEqual(assets[3].ref_width_height, (720, 480))
self.assertEqual(assets[3].dis_width_height, None)
def test_run_strred_fextractor_blackframes(self):
print 'test on running STRRED feature extractor on flat frames...'
ref_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_10.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
from vmaf.core.result_store import FileSystemResultStore
result_store = FileSystemResultStore(logger=None)
self.fextractor = StrredFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=result_store
)
print ' running for the first time with fresh calculation...'
self.fextractor.run(parallelize=True)
result0, result1 = self.fextractor.results
import os
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.fextractor.run(parallelize=True)
results = self.fextractor.results
# ignore NaN
for result in results:
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 1220.5679849999999, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 50983.3097155, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 62228595.6081, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_feature_assembler_selected_atom_feature(self):
print 'test on feature assembler with selected atom features...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature':['vif', 'motion']},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_score']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_score']
def test_with(self):
with YuvReader(
filepath=VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv"),
width=576,
height=324,
yuv_type='yuv420p'
) as yuv_reader:
self.assertEquals(yuv_reader.file.__class__, file)
def test_kflk_metrics_performance(self):
mat_filepath = VmafConfig.test_resource_path('data_Toyama.mat')
mat_dict = scipy.io.loadmat(mat_filepath)
results = KflkPerfMetric._metrics_performance(mat_dict['objScoDif'], mat_dict['signif'])
self.assertAlmostEqual(np.mean(results['AUC_DS']), 0.69767003960902052, places=6)
self.assertAlmostEqual(np.mean(results['AUC_BW']), 0.94454700301894534, places=6)
self.assertAlmostEqual(np.mean(results['CC_0']), 0.88105386206276415, places=6)
self.assertAlmostEqual(np.mean(results['THR']), 6.2392849606450556, places=6)
def test_read_dataset_crop_and_pad(self):
train_dataset_path = VmafConfig.test_resource_path('example_dataset_crop_pad.py')
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
self.assertEquals(len(train_assets), 3)
self.assertEquals(str(train_assets[0]), "example_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81")
self.assertEquals(str(train_assets[1]), "example_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_padiw+100:ih+100:50:50")
self.assertEquals(str(train_assets[2]), "example_0_3_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81_padiw+288:ih+162:144:81")
def test_read_image_dataset_notyuv(self):
dataset_path = VmafConfig.test_resource_path('test_image_dataset_notyuv.py')
dataset = import_python_file(dataset_path)
assets = read_dataset(dataset)
self.assertEquals(len(assets), 4)
self.assertTrue(assets[0].ref_width_height is None)
self.assertTrue(assets[0].dis_width_height is None)
self.assertEquals(assets[0].quality_width_height, (1920, 1080))
def test_run_vmaf_training_raw_dataset(self):
exe = VmafConfig.root_path('run_vmaf_training')
cmd = "{exe} {dataset} {param} {param} {output} --parallelize --suppress-plot".format(
exe=exe,
dataset=self.raw_dataset_filename,
param=self.param_filename,
output=self.out_model_filepath)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
def test_explain_train_test_model(self):
model_class = SklearnRandomForestTrainTestModel
train_dataset_path = VmafConfig.test_resource_path('test_image_dataset_diffdim.py')
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
fextractor = MomentNorefFeatureExtractor(
train_assets,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
fextractor.run(parallelize=True)
self.features = fextractor.results
xys = model_class.get_xys_from_results(self.features[:7])
model = model_class({'norm_type':'normalize', 'random_state':0}, None)
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'],
['Moment_noref_feature_1st_score',
'Moment_noref_feature_2nd_score',
'Moment_noref_feature_var_score']
)
def test_run_vmaf_fextractor_with_cropping(self):
# crop_cmd: 288:162:144:81 - crop to 288x162 with upper-left pixel
# starting at coordinate (144, 81)
ref_path = VmafConfig.test_resource_path("yuv",
"src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"src01_hrc01_576x324.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324,
'crop_cmd': '288:162:144:81',
'quality_width': 288,
'quality_height': 162,
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324,
'crop_cmd': '288:162:144:81',
'quality_width': 288,
'quality_height': 162,
})
self.fextractor = VmafFeatureExtractor([asset, asset_original],
None,
fifo_mode=False)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.45365762500000012,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
2.8779373333333331,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9388824973398119,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
23.942050354166668,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
2.8779373333333331,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
31.71648420833333,
places=4)
class StrredFeatureExtractor(MatlabFeatureExtractor):
TYPE = 'STRRED_feature'
# VERSION = '1.0'
# VERSION = '1.1' # fix matlab code where width and height are mistakenly swapped
# VERSION = '1.2' # fix minor frame and prev frame swap issue
VERSION = '1.3' # align ST-RRED with ST-RREDopt calculations
ATOM_FEATURES = [
'srred',
'trred',
]
DERIVED_ATOM_FEATURES = [
'strred',
]
MATLAB_WORKSPACE = VmafConfig.root_path('matlab', 'strred')
@classmethod
def _assert_an_asset(cls, asset):
super(StrredFeatureExtractor, cls)._assert_an_asset(asset)
assert asset.ref_yuv_type == 'yuv420p' and asset.dis_yuv_type == 'yuv420p', \
'STRRED feature extractor only supports yuv420p for now.'
def _generate_result(self, asset):
# routine to call the command-line executable and generate quality
# scores in the log file.
ref_workfile_path = asset.ref_workfile_path
dis_workfile_path = asset.dis_workfile_path
log_file_path = self._get_log_file_path(asset)
current_dir = os.getcwd() + '/'
ref_workfile_path = make_absolute_path(ref_workfile_path, current_dir)
dis_workfile_path = make_absolute_path(dis_workfile_path, current_dir)
log_file_path = make_absolute_path(log_file_path, current_dir)
quality_width, quality_height = asset.quality_width_height
strred_cmd = '''{matlab} -nodisplay -nosplash -nodesktop -r "run_strred('{ref}', '{dis}', {h}, {w}); exit;" >> {log_file_path}'''.format(
matlab=VmafExternalConfig.get_and_assert_matlab(),
ref=ref_workfile_path,
dis=dis_workfile_path,
w=quality_width,
h=quality_height,
log_file_path=log_file_path,
)
if self.logger:
self.logger.info(strred_cmd)
os.chdir(self.MATLAB_WORKSPACE)
run_process(strred_cmd, shell=True)
os.chdir(current_dir)
@classmethod
def _post_process_result(cls, result):
# override Executor._post_process_result
def _strred(srred_trred):
srred, trred = srred_trred
try:
return srred * trred
except TypeError: # possible either srred or trred is None
return None
result = super(StrredFeatureExtractor,
cls)._post_process_result(result)
# calculate refvar and disvar from ref1st, ref2nd, dis1st, dis2nd
srred_scores_key = cls.get_scores_key('srred')
trred_scores_key = cls.get_scores_key('trred')
strred_scores_key = cls.get_scores_key('strred')
strred_all_same_scores_key = cls.get_scores_key('strred_all_same')
srred_scores = result.result_dict[srred_scores_key]
trred_scores = result.result_dict[trred_scores_key]
assert len(srred_scores) == len(trred_scores)
# === Way One: consistent with VMAF framework, which is to multiply S and T scores per frame, then average
strred_scores = map(_strred, zip(srred_scores, trred_scores))
# === Way Two: authentic way of calculating STRRED score: average first, then multiply ===
strred_all_same_scores = ListStats.nonemean(
srred_scores) * ListStats.nonemean(trred_scores) * np.ones(
len(srred_scores))
result.result_dict[strred_all_same_scores_key] = strred_all_same_scores
result.result_dict[strred_scores_key] = strred_scores
# validate
for feature in cls.DERIVED_ATOM_FEATURES:
assert cls.get_scores_key(feature) in result.result_dict
return result
dataset_name = 'example'
yuv_fmt = 'yuv420p'
width = 1920
height = 1080
ref_score = 100.0
from vmaf.config import VmafConfig
ref_videos = [
{
'content_id':
0,
'path':
VmafConfig.test_resource_path('yuv',
'checkerboard_1920_1080_10_3_0_0.yuv')
},
{
'content_id': 1,
'path': VmafConfig.test_resource_path('yuv', 'flat_1920_1080_0.yuv')
},
]
dis_videos = [
{
'content_id':
0,
'asset_id':
0,
'os': [100, 100, 100, 100, 100],
'path':
class EnsembleVmafQualityRunner(VmafQualityRunner):
TYPE = 'EnsembleVMAF'
VERSION = '{}-Ensemble'.format(VmafQualityRunner.VERSION)
DEFAULT_MODEL_FILEPATH = [
VmafConfig.model_path("vmaf_v0.6.1.pkl"),
VmafConfig.model_path("vmaf_v0.6.1.pkl")
]
# this now needs to become a list
DEFAULT_FEATURE_DICT = [{
'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']
}, {
'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']
}]
def _populate_result_dict(self, feature_result, pred_result, result_dict):
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = pred_result[
'ys_pred'] # add quality score
return result_dict
def _get_ensemblevmaf_feature_assembler_instance(self, asset):
# load TrainTestModel only to retrieve its 'feature_dict' extra info
ensem_models = self._load_model(asset)
ensemblevmaf_fassemblers = []
for model_ind, model_now in enumerate(ensem_models):
feature_dict = model_now.get_appended_info('feature_dict')
if feature_dict is None:
feature_dict = self.DEFAULT_FEATURE_DICT[model_ind]
ensemblevmaf_fassembler = FeatureAssembler(
feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
ensemblevmaf_fassemblers.append(ensemblevmaf_fassembler)
return ensemblevmaf_fassemblers
def _get_Nframes(self, pred_result):
return len(pred_result['ys_pred'])
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
ensemblevmaf_fassemblers = self._get_ensemblevmaf_feature_assembler_instance(
asset)
# each model is associated with a Feature Assembler
Nmodels = len(ensemblevmaf_fassemblers)
pred_result_ensem_models = []
result_dict = {}
for model_ind in range(Nmodels):
evmaf_fassembler = ensemblevmaf_fassemblers[model_ind]
evmaf_fassembler.run()
feature_result = evmaf_fassembler.results[0]
model = self._load_model(asset)[model_ind]
xs = model.get_per_unit_xs_from_a_result(feature_result)
if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
disable_clip_score = self.optional_dict['disable_clip_score']
else:
disable_clip_score = False
if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
enable_transform_score = self.optional_dict[
'enable_transform_score']
else:
enable_transform_score = False
pred_result = self.predict_with_model(
model,
xs,
disable_clip_score=disable_clip_score,
enable_transform_score=enable_transform_score)
result_dict = self._populate_result_dict(feature_result,
pred_result, result_dict)
pred_result_ensem_models.append(pred_result)
assert Nmodels > 0
Nframes = self._get_Nframes(pred_result)
all_model_scores = np.zeros((Nmodels, Nframes))
all_model_score_names = self.ensemblevmaf_get_scores_key(Nmodels)
for model_ind in range(Nmodels):
result_dict[all_model_score_names[
model_ind]] = pred_result_ensem_models[model_ind][
'ys_pred'] # add quality score
all_model_scores[
model_ind, :] = pred_result_ensem_models[model_ind]['ys_pred']
# perform prediction averaging (simple average for now)
pred_result_ensem_models_aggregate = np.mean(all_model_scores, axis=0)
# write results
result_dict[self.get_scores_key()] = pred_result_ensem_models_aggregate
return Result(asset, self.executor_id, result_dict)
def ensemblevmaf_get_scores_key(self, Nmodels):
scores_name_list = []
for model_ind in range(Nmodels):
scores_name_list.append(self.TYPE + '_model_' + str(model_ind) +
'_scores')
return scores_name_list
def _load_model(self, asset):
if self.optional_dict is not None \
and 'model_filepath' in self.optional_dict \
and self.optional_dict['model_filepath'] is not None:
model_filepath = self.optional_dict['model_filepath']
else:
model_filepath = self.DEFAULT_MODEL_FILEPATH
model = []
for model_filepath_part in model_filepath:
model.append(
TrainTestModel.from_file(model_filepath_part, self.logger))
return model
def _remove_result(self, asset):
# Override Executor._remove_result(self, asset) by redirecting it to the
# FeatureAssembler.
ensemblevmaf_fassemblers = self._get_ensemblevmaf_feature_assembler_instance(
asset)
for ensemblevmaf_fassembler in ensemblevmaf_fassemblers:
ensemblevmaf_fassembler.remove_results()
class STMADFeatureExtractor(MatlabFeatureExtractor):
TYPE = "STMAD_feature"
VERSION = "0.1"
ATOM_FEATURES = ['smad', 'tmad', 'stmad']
DERIVED_ATOM_FEATURES = [
'smad_all_same', 'tmad_all_same', 'stmad_all_same'
]
MATLAB_WORKSPACE = VmafConfig.root_path('matlab', 'STMAD_2011_MatlabCode')
# compile necessary functions; need to use mex from within matlab
def _custom_init(self):
def run_stmad_cmd(stmad_cmd):
current_dir = os.getcwd() + '/'
os.chdir(self.MATLAB_WORKSPACE)
run_process(stmad_cmd, shell=True)
os.chdir(current_dir)
stmad_mex_cmd_1 = '''{matlab} -nodisplay -nosplash -nodesktop -r "mex ical_std.c; exit;"'''.format(
matlab=VmafExternalConfig.get_and_assert_matlab(), )
run_stmad_cmd(stmad_mex_cmd_1)
stmad_mex_cmd_2 = '''{matlab} -nodisplay -nosplash -nodesktop -r "mex ical_stat.c; exit;"'''.format(
matlab=VmafExternalConfig.get_and_assert_matlab(), )
run_stmad_cmd(stmad_mex_cmd_2)
@classmethod
def _assert_an_asset(cls, asset):
super(STMADFeatureExtractor, cls)._assert_an_asset(asset)
assert asset.ref_yuv_type == 'yuv420p' and asset.dis_yuv_type == 'yuv420p', \
'STMAD feature extractor only supports yuv420p for now.'
def _generate_result(self, asset):
# routine to call the command-line executable and generate quality
# scores in the log file.
ref_workfile_path = asset.ref_workfile_path
dis_workfile_path = asset.dis_workfile_path
log_file_path = self._get_log_file_path(asset)
current_dir = os.getcwd() + '/'
ref_workfile_path = make_absolute_path(ref_workfile_path, current_dir)
dis_workfile_path = make_absolute_path(dis_workfile_path, current_dir)
log_file_path = make_absolute_path(log_file_path, current_dir)
quality_width, quality_height = asset.quality_width_height
stmad_cmd = '''{matlab} -nodisplay -nosplash -nodesktop -r "run_stmad('{ref}', '{dis}', {w}, {h}); exit;" >> {log_file_path}'''.format(
matlab=VmafExternalConfig.get_and_assert_matlab(),
ref=ref_workfile_path,
dis=dis_workfile_path,
w=quality_width,
h=quality_height,
log_file_path=log_file_path,
)
if self.logger:
self.logger.info(stmad_cmd)
os.chdir(self.MATLAB_WORKSPACE)
run_process(stmad_cmd, shell=True)
os.chdir(current_dir)
@classmethod
def _post_process_result(cls, result):
# override Executor._post_process_result
result = super(STMADFeatureExtractor, cls)._post_process_result(result)
smad_scores_key = cls.get_scores_key('smad')
tmad_scores_key = cls.get_scores_key('tmad')
stmad_scores_key = cls.get_scores_key('stmad')
smad_all_same_scores_key = cls.get_scores_key('smad_all_same')
tmad_all_same_scores_key = cls.get_scores_key('tmad_all_same')
stmad_all_same_scores_key = cls.get_scores_key('stmad_all_same')
# remember that smad, tmad, stmad are only one value for the whole video so
# stmad_all_same_scores is identical to stmad_scores
smad_scores = result.result_dict[smad_scores_key]
tmad_scores = result.result_dict[tmad_scores_key]
stmad_scores = result.result_dict[stmad_scores_key]
result.result_dict[smad_scores_key] = smad_scores
result.result_dict[tmad_scores_key] = tmad_scores
result.result_dict[stmad_scores_key] = stmad_scores
result.result_dict[smad_all_same_scores_key] = smad_scores
result.result_dict[tmad_all_same_scores_key] = tmad_scores
result.result_dict[stmad_all_same_scores_key] = stmad_scores
# validate
for feature in cls.DERIVED_ATOM_FEATURES:
assert cls.get_scores_key(feature) in result.result_dict
return result
class VmafLegacyQualityRunner(QualityRunner):
TYPE = 'VMAF_legacy'
# VERSION = '1.1'
VERSION = 'F' + VmafFeatureExtractor.VERSION + '-1.1'
FEATURE_ASSEMBLER_DICT = {'VMAF_feature': 'all'}
FEATURE_RESCALE_DICT = {'VMAF_feature_vif_scores': (0.0, 1.0),
'VMAF_feature_adm_scores': (0.4, 1.0),
'VMAF_feature_ansnr_scores': (10.0, 50.0),
'VMAF_feature_motion_scores': (0.0, 20.0)}
SVM_MODEL_FILE = VmafConfig.model_path("model_V8a.model")
# model_v8a.model is trained with customized feature order:
SVM_MODEL_ORDERED_SCORES_KEYS = ['VMAF_feature_vif_scores',
'VMAF_feature_adm_scores',
'VMAF_feature_ansnr_scores',
'VMAF_feature_motion_scores']
def _get_quality_scores(self, asset):
raise NotImplementedError
def _generate_result(self, asset):
raise NotImplementedError
def _get_vmaf_feature_assembler_instance(self, asset):
vmaf_fassembler = FeatureAssembler(
feature_dict=self.FEATURE_ASSEMBLER_DICT,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
return vmaf_fassembler
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
# =====================================================================
# SVR predict
model = svmutil.svm_load_model(self.SVM_MODEL_FILE)
ordered_scaled_scores_list = []
for scores_key in self.SVM_MODEL_ORDERED_SCORES_KEYS:
scaled_scores = self._rescale(feature_result[scores_key],
self.FEATURE_RESCALE_DICT[scores_key])
ordered_scaled_scores_list.append(scaled_scores)
scores = []
for score_vector in zip(*ordered_scaled_scores_list):
vif, adm, ansnr, motion = score_vector
xs = [[vif, adm, ansnr, motion]]
score = svmutil.svm_predict([0], xs, model)[0][0]
score = self._post_correction(motion, score)
scores.append(score)
result_dict = {}
# add all feature result
result_dict.update(feature_result.result_dict)
# add quality score
result_dict[self.get_scores_key()] = scores
return Result(asset, self.executor_id, result_dict)
def _post_correction(self, motion, score):
# post-SVM correction
if motion > 12.0:
val = motion
if val > 20.0:
val = 20
score *= ((val - 12) * 0.015 + 1)
if score > 100.0:
score = 100.0
elif score < 0.0:
score = 0.0
return score
@classmethod
def _rescale(cls, vals, lower_upper_bound):
lower_bound, upper_bound = lower_upper_bound
vals = np.double(vals)
vals = np.clip(vals, lower_bound, upper_bound)
vals = (vals - lower_bound) / (upper_bound - lower_bound)
return vals
# override
def _remove_result(self, asset):
# Override Executor._remove_result by redirecting it to the
# FeatureAssembler.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.remove_results()
def setUp(self):
self.raw_dataset_filepath = VmafConfig.resource_path("dataset", "NFLX_dataset_public_raw.py")
self.derived_dataset_path = VmafConfig.workdir_path("test_derived_dataset.py")
self.derived_dataset_path_pyc = VmafConfig.workdir_path("test_derived_dataset.pyc")
def test_load_result_with_none(self):
result = FileSystemResultStore.load_result(VmafConfig.test_resource_path('result_with_none.txt'))
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(result['STRRED_feature_srred_score'], 5829.2644469999996, places=4)
def main():
if len(sys.argv) < 5:
print_usage()
return 2
try:
q_width = int(sys.argv[1])
q_height = int(sys.argv[2])
ref_file = sys.argv[3]
dis_file = sys.argv[4]
except ValueError:
print_usage()
return 2
if q_width < 0 or q_height < 0:
print(
"quality_width and quality_height must be non-negative, but are {w} and {h}"
.format(w=q_width, h=q_height))
print_usage()
return 2
model_path = get_cmd_option(sys.argv, 5, len(sys.argv), '--model')
out_fmt = get_cmd_option(sys.argv, 5, len(sys.argv), '--out-fmt')
if not (out_fmt is None or out_fmt == 'xml' or out_fmt == 'json'
or out_fmt == 'text'):
print_usage()
return 2
ref_fmt = get_cmd_option(sys.argv, 5, len(sys.argv), '--ref-fmt')
if not (ref_fmt is None or ref_fmt in FMTS):
print('--ref-fmt can only have option among {}'.format(
', '.join(FMTS)))
ref_width = get_cmd_option(sys.argv, 5, len(sys.argv), '--ref-width')
ref_height = get_cmd_option(sys.argv, 5, len(sys.argv), '--ref-height')
dis_width = get_cmd_option(sys.argv, 5, len(sys.argv), '--dis-width')
dis_height = get_cmd_option(sys.argv, 5, len(sys.argv), '--dis-height')
obj_file = get_cmd_option(sys.argv, 6, len(sys.argv), '--attention')
dis_fmt = get_cmd_option(sys.argv, 5, len(sys.argv), '--dis-fmt')
if not (dis_fmt is None or dis_fmt in FMTS):
print('--dis-fmt can only have option among {}'.format(
', '.join(FMTS)))
work_dir = get_cmd_option(sys.argv, 5, len(sys.argv), '--work-dir')
pool_method = get_cmd_option(sys.argv, 5, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print('--pool can only have option among {}'.format(
', '.join(POOL_METHODS)))
return 2
show_local_explanation = cmd_option_exists(sys.argv, 5, len(sys.argv),
'--local-explain')
phone_model = cmd_option_exists(sys.argv, 5, len(sys.argv),
'--phone-model')
enable_conf_interval = cmd_option_exists(sys.argv, 5, len(sys.argv),
'--ci')
save_plot_dir = get_cmd_option(sys.argv, 5, len(sys.argv), '--save-plot')
if work_dir is None:
work_dir = VmafConfig.workdir_path()
asset_dict = {'quality_width': q_width, 'quality_height': q_height}
if ref_fmt is None:
asset_dict['ref_yuv_type'] = 'notyuv'
else:
if ref_width is None or ref_height is None:
print(
'if --ref-fmt is specified, both --ref-width and --ref-height must be specified'
)
return 2
else:
asset_dict['ref_yuv_type'] = ref_fmt
asset_dict['ref_width'] = ref_width
asset_dict['ref_height'] = ref_height
if dis_fmt is None:
asset_dict['dis_yuv_type'] = 'notyuv'
else:
if dis_width is None or dis_height is None:
print(
'if --dis-fmt is specified, both --dis-width and --dis-height must be specified'
)
return 2
else:
asset_dict['dis_yuv_type'] = dis_fmt
asset_dict['dis_width'] = dis_width
asset_dict['dis_height'] = dis_height
if show_local_explanation and enable_conf_interval:
print('cannot set both --local-explain and --ci flags')
return 2
asset = Asset(
dataset="cmd",
content_id=abs(hash(get_file_name_without_extension(ref_file))) %
(10**16),
asset_id=abs(hash(get_file_name_without_extension(ref_file))) %
(10**16),
workdir_root=work_dir,
ref_path=ref_file,
dis_path=dis_file,
obj_path=obj_file,
asset_dict=asset_dict,
)
assets = [asset]
if show_local_explanation:
from vmaf.core.quality_runner_extra import VmafQualityRunnerWithLocalExplainer
runner_class = VmafQualityRunnerWithLocalExplainer
elif enable_conf_interval:
from vmaf.core.quality_runner import BootstrapVmafQualityRunner
runner_class = BootstrapVmafQualityRunner
else:
runner_class = VmafQualityRunner
if model_path is None:
optional_dict = None
else:
optional_dict = {'model_filepath': model_path}
if phone_model:
if optional_dict is None:
optional_dict = {}
optional_dict['enable_transform_score'] = True
runner = runner_class(
assets,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=optional_dict,
optional_dict2=None,
)
# run
runner.run()
result = runner.results[0]
# pooling
if pool_method == 'harmonic_mean':
result.set_score_aggregate_method(ListStats.harmonic_mean)
elif pool_method == 'min':
result.set_score_aggregate_method(np.min)
elif pool_method == 'median':
result.set_score_aggregate_method(np.median)
elif pool_method == 'perc5':
result.set_score_aggregate_method(ListStats.perc5)
elif pool_method == 'perc10':
result.set_score_aggregate_method(ListStats.perc10)
elif pool_method == 'perc20':
result.set_score_aggregate_method(ListStats.perc20)
else: # None or 'mean'
pass
# output
if out_fmt == 'xml':
print(result.to_xml())
elif out_fmt == 'json':
print(result.to_json())
else: # None or 'text'
print(str(result))
# local explanation
if show_local_explanation:
runner.show_local_explanations([result])
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
return 0
def setUp(self):
self.output_model_filepath = VmafConfig.workspace_path(
"model", "test_output_model.pkl")
def read_dataset(dataset, **kwargs):
groundtruth_key = kwargs[
'groundtruth_key'] if 'groundtruth_key' in kwargs else None
skip_asset_with_none_groundtruth = kwargs['skip_asset_with_none_groundtruth'] \
if 'skip_asset_with_none_groundtruth' in kwargs else False
# asserts, can add more to the list...
assert hasattr(dataset, 'dataset_name')
assert hasattr(dataset, 'ref_videos')
assert hasattr(dataset, 'dis_videos')
assert hasattr(dataset, 'yuv_fmt') or all(
['yuv_fmt' in ref_video for ref_video in dataset.ref_videos])
data_set_name = dataset.dataset_name
ref_videos = dataset.ref_videos
dis_videos = dataset.dis_videos
width = dataset.width if hasattr(dataset, 'width') else None
height = dataset.height if hasattr(dataset, 'height') else None
yuv_fmt = dataset.yuv_fmt if hasattr(dataset, 'yuv_fmt') else None
quality_width = dataset.quality_width if hasattr(dataset,
'quality_width') else None
quality_height = dataset.quality_height if hasattr(
dataset, 'quality_height') else None
resampling_type = dataset.resampling_type if hasattr(
dataset, 'resampling_type') else None
crop_cmd = dataset.crop_cmd if hasattr(dataset, 'crop_cmd') else None
pad_cmd = dataset.pad_cmd if hasattr(dataset, 'pad_cmd') else None
workfile_yuv_type = dataset.workfile_yuv_type if hasattr(
dataset, 'workfile_yuv_type') else None
duration_sec = dataset.duration_sec if hasattr(dataset,
'duration_sec') else None
ref_dict = {} # dictionary of content_id -> path for ref videos
for ref_video in ref_videos:
ref_dict[ref_video['content_id']] = ref_video
assets = []
for dis_video in dis_videos:
if groundtruth_key is not None:
groundtruth = dis_video[groundtruth_key]
else:
if 'dmos' in dis_video:
groundtruth = dis_video['dmos']
elif 'mos' in dis_video:
groundtruth = dis_video['mos']
elif 'groundtruth' in dis_video:
groundtruth = dis_video['groundtruth']
else:
groundtruth = None
if 'os' in dis_video:
raw_groundtruth = dis_video['os']
else:
raw_groundtruth = None
if 'groundtruth_std' in dis_video:
groundtruth_std = dis_video['groundtruth_std']
else:
groundtruth_std = None
if 'fps' in dis_video:
fps = dis_video['fps']
else:
fps = None
if 'rebuf_indices' in dis_video:
rebuf_indices = dis_video['rebuf_indices']
else:
rebuf_indices = None
ref_video = ref_dict[dis_video['content_id']]
ref_path = ref_video['path']
ref_yuv_fmt_ = yuv_fmt if yuv_fmt is not None else ref_dict[
dis_video['content_id']]['yuv_fmt']
dis_yuv_fmt_ = dis_video[
'yuv_fmt'] if 'yuv_fmt' in dis_video else ref_yuv_fmt_
if width is not None:
width_ = width
elif 'width' in ref_video:
width_ = ref_video[
'width'] # NOTE: width in ref_video not dis_video
else:
width_ = None
if height is not None:
height_ = height
elif 'height' in ref_video:
height_ = ref_video[
'height'] # NOTE: height in ref_video not dis_video
else:
height_ = None
if quality_width is not None:
quality_width_ = quality_width
elif 'quality_width' in dis_video:
quality_width_ = dis_video['quality_width']
else:
quality_width_ = None
if quality_height is not None:
quality_height_ = quality_height
elif 'quality_height' in dis_video:
quality_height_ = dis_video['quality_height']
else:
quality_height_ = None
if resampling_type is not None:
resampling_type_ = resampling_type
elif 'resampling_type' in dis_video:
resampling_type_ = dis_video['resampling_type']
else:
resampling_type_ = None
if crop_cmd is not None:
crop_cmd_ = crop_cmd
elif 'crop_cmd' in dis_video:
crop_cmd_ = dis_video['crop_cmd']
else:
crop_cmd_ = None
if pad_cmd is not None:
pad_cmd_ = pad_cmd
elif 'pad_cmd' in dis_video:
pad_cmd_ = dis_video['pad_cmd']
else:
pad_cmd_ = None
if duration_sec is not None:
duration_sec_ = duration_sec
elif 'duration_sec' in dis_video:
duration_sec_ = dis_video['duration_sec']
else:
duration_sec_ = None
asset_dict = {
'ref_yuv_type': ref_yuv_fmt_,
'dis_yuv_type': dis_yuv_fmt_
}
if width_ is not None:
if asset_dict['ref_yuv_type'] != 'notyuv':
asset_dict['ref_width'] = width_
if asset_dict['dis_yuv_type'] != 'notyuv':
asset_dict['dis_width'] = width_
if height_ is not None:
if asset_dict['ref_yuv_type'] != 'notyuv':
asset_dict['ref_height'] = height_
if asset_dict['dis_yuv_type'] != 'notyuv':
asset_dict['dis_height'] = height_
if groundtruth is not None:
asset_dict['groundtruth'] = groundtruth
if raw_groundtruth is not None:
asset_dict['raw_groundtruth'] = raw_groundtruth
if groundtruth_std is not None:
asset_dict['groundtruth_std'] = groundtruth_std
if quality_width_ is not None:
asset_dict['quality_width'] = quality_width_
if quality_height_ is not None:
asset_dict['quality_height'] = quality_height_
if resampling_type_ is not None:
asset_dict['resampling_type'] = resampling_type_
if crop_cmd_ is not None:
asset_dict['crop_cmd'] = crop_cmd_
if pad_cmd_ is not None:
asset_dict['pad_cmd'] = pad_cmd_
if duration_sec_ is not None:
asset_dict['duration_sec'] = duration_sec_
if workfile_yuv_type is not None:
asset_dict['workfile_yuv_type'] = workfile_yuv_type
if fps is not None:
asset_dict['fps'] = fps
if rebuf_indices is not None:
asset_dict['rebuf_indices'] = rebuf_indices
if groundtruth is None and skip_asset_with_none_groundtruth:
pass
else:
asset = Asset(
dataset=data_set_name,
content_id=dis_video['content_id'],
asset_id=dis_video['asset_id'],
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_video['path'],
asset_dict=asset_dict,
)
assets.append(asset)
return assets
def test_run_testlibdyn_runner(self):
print('test on running TESTLIBDYN runner...')
ref_path = VmafConfig.test_resource_path("yuv",
"src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"src01_hrc01_576x324.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.runner = LibDynRunner(
[asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
)
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale0_score'],
0.363420458333,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale1_score'],
0.766647520833,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale2_score'],
0.862854708333,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_vif_scale3_score'],
0.915971791667,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_motion2_score'],
3.8953518541666665,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_adm2_score'],
0.93458777083333333,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_psnr_score'],
30.7550666667,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_ssim_score'],
0.86322654166666657,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_ms_ssim_score'],
0.9632498125,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale0_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale1_score'],
0.999999958333,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale2_score'],
0.999999416667,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_vif_scale3_score'],
0.999999208333,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_motion2_score'],
3.8953518541666665,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_adm2_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_psnr_score'],
60.0,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_ssim_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['TESTLIBDYN_ms_ssim_score'],
1.0,
places=4)
self.assertAlmostEqual(results[0]['TESTLIBDYN_score'],
76.699271272486044,
places=3)
self.assertAlmostEqual(results[1]['TESTLIBDYN_score'],
99.946416604585025,
places=4)
def test_run_vmaf_fextractor_with_cropping_and_padding_to_original_wh_proc(
self):
# crop_cmd: 288:162:144:81 - crop to the center 288x162 image
# pad_cmd: iw+288:ih+162:144:81 - pad back to the original size
ref_path = VmafConfig.test_resource_path("yuv",
"src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"src01_hrc01_576x324.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324,
'crop_cmd': '288:162:144:81',
'pad_cmd': 'iw+288:ih+162:144:81',
'quality_width': 576,
'quality_height': 324,
'ref_proc_callback': 'identity',
'dis_proc_callback': 'identity',
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324,
'crop_cmd': '288:162:144:81',
'pad_cmd': 'iw+288:ih+162:144:81',
'quality_width': 576,
'quality_height': 324,
})
self.fextractor = VmafFeatureExtractor([asset, asset_original],
None,
fifo_mode=True)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.64106379166666672,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
0.7203213958333331,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9469305256822512,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
32.78451041666667,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
0.7203213958333331,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
40.280504208333333,
places=4)
def test_run_vmaf_fextractor_with_padding(self):
# pad_cmd: iw+100:ih+100:50:50 - pad to (iw+100)x(ih+100), where iw is
# input width, ih is input height, and starting point is (-50, -50)
ref_path = VmafConfig.test_resource_path("yuv",
"src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"src01_hrc01_576x324.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324,
'pad_cmd': 'iw+100:ih+100:50:50',
'quality_width': 676,
'quality_height': 424,
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324,
'pad_cmd': 'iw+100:ih+100:50:50',
'quality_width': 676,
'quality_height': 424,
})
self.fextractor = VmafFeatureExtractor([asset, asset_original],
None,
fifo_mode=True)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.51023564583333325,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
2.6397702083333332,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9410537302204777,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
26.893242291666667,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
2.6397702083333332,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
34.306043416666668,
places=4)
class VmafossExecQualityRunner(QualityRunner):
TYPE = 'VMAFOSSEXEC'
# VERSION = '0.3'
# DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv3.pkl.model")
# VERSION = '0.3.1'
# DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv3a.pkl.model")
# VERSION = '0.3.2'
# ALGO_VERSION = 0
# # DEFAULT_MODEL_FILEPATH_DOTMODEL = VmafConfig.model_path("nflxall_vmafv4.pkl.model")
# DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv4.pkl")
VERSION = 'F' + VmafFeatureExtractor.VERSION + '-0.6.1'
ALGO_VERSION = 2
# trained with resource/param/vmaf_v6.py on private/user/zli/resource/dataset/dataset/derived/vmafplusstudy_laptop_raw_generalandcornercase.py, MLER, y=x+17
DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_v0.6.1.pkl")
FEATURES = ['adm2', 'adm_scale0', 'adm_scale1', 'adm_scale2', 'adm_scale3',
'motion', 'vif_scale0', 'vif_scale1', 'vif_scale2',
'vif_scale3', 'vif', 'psnr', 'ssim', 'ms_ssim', 'motion2']
@classmethod
def get_feature_scores_key(cls, atom_feature):
return "{type}_{atom_feature}_scores".format(
type=cls.TYPE, atom_feature=atom_feature)
def _generate_result(self, asset):
# routine to call the command-line executable and generate quality
# scores in the log file.
log_file_path = self._get_log_file_path(asset)
if self.optional_dict is not None \
and 'model_filepath' in self.optional_dict \
and self.optional_dict['model_filepath'] is not None:
model_filepath = self.optional_dict['model_filepath']
else:
model_filepath = self.DEFAULT_MODEL_FILEPATH
if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
disable_clip_score = self.optional_dict['disable_clip_score']
else:
disable_clip_score = False
if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
enable_transform_score = self.optional_dict['enable_transform_score']
else:
enable_transform_score = False
if self.optional_dict is not None and 'phone_model' in self.optional_dict:
phone_model = self.optional_dict['phone_model']
else:
phone_model = False
if self.optional_dict is not None and 'disable_avx' in self.optional_dict:
disable_avx = self.optional_dict['disable_avx']
else:
disable_avx = False
quality_width, quality_height = asset.quality_width_height
fmt=self._get_workfile_yuv_type(asset)
w=quality_width
h=quality_height
ref_path=asset.ref_workfile_path
dis_path=asset.dis_workfile_path
model=model_filepath
exe=self._get_exec()
logger = self.logger
ExternalProgramCaller.call_vmafossexec(fmt, w, h, ref_path, dis_path, model, log_file_path,
disable_clip_score, enable_transform_score,
phone_model, disable_avx, exe, logger)
def _get_exec(self):
return None # signaling default
def _get_quality_scores(self, asset):
# routine to read the quality scores from the log file, and return
# the scores in a dictionary format.
log_file_path = self._get_log_file_path(asset)
tree = ElementTree.parse(log_file_path)
root = tree.getroot()
scores = []
feature_scores = [[] for _ in self.FEATURES]
for frame in root.findall('frames/frame'):
scores.append(float(frame.attrib['vmaf']))
for i_feature, feature in enumerate(self.FEATURES):
try:
feature_scores[i_feature].append(float(frame.attrib[feature]))
except KeyError:
pass # some features may be missing
assert len(scores) != 0
quality_result = {
self.get_scores_key(): scores,
}
for i_feature, feature in enumerate(self.FEATURES):
if len(feature_scores[i_feature]) != 0:
quality_result[self.get_feature_scores_key(feature)] = feature_scores[i_feature]
return quality_result
from vmaf.config import VmafConfig
dataset_name = 'test_image'
yuv_fmt = 'yuv444p'
dataset_dir = VmafConfig.test_resource_path('test_image_yuv')
quality_width = 200
quality_height = 100
resampling_type = 'bicubic'
ref_videos = [
{'content_id': 0,
'content_name': '100007',
'height': 321,
'path': dataset_dir + '/100007.yuv',
'width': 481},
{'content_id': 1,
'content_name': '100039',
'height': 321,
'path': dataset_dir + '/100039.yuv',
'width': 481},
{'content_id': 2,
'content_name': '100075',
'height': 321,
'path': dataset_dir + '/100075.yuv',
'width': 481},
{'content_id': 4,
'content_name': '100098',
'height': 321,
'path': dataset_dir + '/100098.yuv',
class VmafQualityRunner(QualityRunner):
TYPE = 'VMAF'
# VERSION = '0.1' # using model nflxall_vmafv1.pkl, VmafFeatureExtractor VERSION 0.1
# DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv1.pkl") # trained with resource/param/vmaf_v1.py on private/resource/dataset/NFLX_dataset.py (30 subjects)
# VERSION = '0.2' # using model nflxall_vmafv2.pkl, VmafFeatureExtractor VERSION 0.2.1
# DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv2.pkl") # trained with resource/param/vmaf_v2.py on private/resource/dataset/NFLX_dataset.py (30 subjects)
# VERSION = '0.3' # using model nflxall_vmafv3.pkl, VmafFeatureExtractor VERSION 0.2.1
# DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv3.pkl") # trained with resource/param/vmaf_v3.py on private/resource/dataset/NFLX_dataset.py (30 subjects)
# VERSION = '0.3.1' # using model nflxall_vmafv3.pkl, VmafFeatureExtractor VERSION 0.2.1, NFLX_dataset with 26 subjects (last 4 outliers removed)
# DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv3a.pkl") # trained with resource/param/vmaf_v3.py on private/resource/dataset/NFLX_dataset.py (26 subjects)
# VERSION = '0.3.2' # using model nflxall_vmafv4.pkl, VmafFeatureExtractor VERSION 0.2.2, NFLX_dataset with 26 subjects (last 4 outliers removed)
# ALGO_VERSION = 0
# DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("nflxall_vmafv4.pkl") # trained with resource/param/vmaf_v4.py on private/resource/dataset/NFLX_dataset.py (26 subjects)
# using model vmaf_v0.6.1.pkl. VmafFeatureExtractor VERSION 0.2.4b.
VERSION = 'F' + VmafFeatureExtractor.VERSION + '-0.6.1'
ALGO_VERSION = 2
# trained with resource/param/vmaf_v6.py on private/user/zli/resource/dataset/dataset/derived/vmafplusstudy_laptop_raw_generalandcornercase.py, MLER, y=x+17
DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_v0.6.1.pkl")
DEFAULT_FEATURE_DICT = {'VMAF_feature': ['vif', 'adm', 'motion', 'ansnr']} # for backward-compatible with older model only
def _get_quality_scores(self, asset):
raise NotImplementedError
def _generate_result(self, asset):
raise NotImplementedError
def _get_vmaf_feature_assembler_instance(self, asset):
# load TrainTestModel only to retrieve its 'feature_dict' extra info
feature_dict = self._load_model(asset).get_appended_info('feature_dict')
if feature_dict is None:
feature_dict = self.DEFAULT_FEATURE_DICT
vmaf_fassembler = FeatureAssembler(
feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
return vmaf_fassembler
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
model = self._load_model(asset)
xs = model.get_per_unit_xs_from_a_result(feature_result)
if self.optional_dict is not None and 'disable_clip_score' in self.optional_dict:
disable_clip_score = self.optional_dict['disable_clip_score']
else:
disable_clip_score = False
if self.optional_dict is not None and 'enable_transform_score' in self.optional_dict:
enable_transform_score = self.optional_dict['enable_transform_score']
else:
enable_transform_score = False
pred_result = self.predict_with_model(model, xs,
disable_clip_score=disable_clip_score,
enable_transform_score=enable_transform_score)
result_dict = self._populate_result_dict(feature_result, pred_result)
return Result(asset, self.executor_id, result_dict)
def _populate_result_dict(self, feature_result, pred_result):
result_dict = {}
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = pred_result['ys_pred'] # add quality score
return result_dict
@classmethod
def predict_with_model(cls, model, xs, **kwargs):
ys_pred = model.predict(xs)['ys_label_pred']
do_transform_score = cls._do_transform_score(kwargs)
if do_transform_score:
ys_pred = cls.transform_score(model, ys_pred)
else:
pass
if 'disable_clip_score' in kwargs and kwargs['disable_clip_score'] is True:
pass
else:
ys_pred = cls.clip_score(model, ys_pred)
return {'ys_pred': ys_pred}
@staticmethod
def _do_transform_score(kwargs):
return 'enable_transform_score' in kwargs and kwargs['enable_transform_score'] is True
@staticmethod
def set_transform_score(model, score_transform):
model.append_info('score_transform', score_transform)
@staticmethod
def set_clip_score(model, score_clip):
model.append_info('score_clip', score_clip)
@staticmethod
def transform_score(model, ys_pred):
"""
Do post processing: transform final quality score e.g. via polynomial
{'p0': 1, 'p1': 1, 'p2': 0.5} means transform through 1 + x + 0.5 * x^2.
For now, only support polynomail up to 2nd-order.
"""
transform_dict = model.get_appended_info('score_transform')
if transform_dict is None:
return ys_pred
y_in = ys_pred
y_out = np.zeros(ys_pred.shape)
# quadratic transform
if 'p0' in transform_dict:
y_out += transform_dict['p0']
if 'p1' in transform_dict:
y_out += transform_dict['p1'] * y_in
if 'p2' in transform_dict:
y_out += transform_dict['p2'] * y_in * y_in
# rectification
if 'out_lte_in' in transform_dict and transform_dict['out_lte_in'] == 'true':
# output must be less than or equal to input
y_out = np.minimum(y_out, y_in)
if 'out_gte_in' in transform_dict and transform_dict['out_gte_in'] == 'true':
# output must be greater than or equal to input
y_out = np.maximum(y_out, y_in)
return y_out
@staticmethod
def clip_score(model, ys_pred):
"""
Do post processing: clip final quality score within e.g. [0, 100]
"""
score_clip = model.get_appended_info('score_clip')
if score_clip is not None:
lb, ub = score_clip
ys_pred = np.clip(ys_pred, lb, ub)
return ys_pred
def _load_model(self, asset):
if self.optional_dict is not None \
and 'model_filepath' in self.optional_dict \
and self.optional_dict['model_filepath'] is not None:
model_filepath = self.optional_dict['model_filepath']
else:
model_filepath = self.DEFAULT_MODEL_FILEPATH
train_test_model_class = self.get_train_test_model_class()
model = train_test_model_class.from_file(model_filepath, self.logger)
return model
def get_train_test_model_class(self):
return LibsvmNusvrTrainTestModel
def _remove_result(self, asset):
# Override Executor._remove_result(self, asset) by redirecting it to the
# FeatureAssembler.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.remove_results()
def test_run_vmafrc_runner_yuv422p10le(self):
ref_path = VmafConfig.test_resource_path(
"yuv", "src01_hrc00_576x324.yuv422p10le.yuv")
dis_path = VmafConfig.test_resource_path(
"yuv", "src01_hrc01_576x324.yuv422p10le.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324,
'yuv_type': 'yuv422p10le'
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324,
'yuv_type': 'yuv422p10le'
})
self.runner = VmafrcQualityRunner([asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'psnr': True,
'ssim': True,
'ms_ssim': True,
})
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale0_score'],
0.3634208125,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale1_score'],
0.7666474166666667,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale2_score'],
0.8628533333333334,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale3_score'],
0.9159719583333334,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_motion2_score'],
3.895352291666667,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_adm2_score'],
0.9345877291666667,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_float_psnr_score'],
30.780577083333331,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_float_ssim_score'],
0.86322654166666657,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_float_ms_ssim_score'],
0.9632406874999999,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale0_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale1_score'],
0.9999998541666666,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale2_score'],
0.9999996041666667,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale3_score'],
0.9999991458333334,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_motion2_score'],
3.895352291666667,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFRC_float_psnr_score'],
72.0,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_float_ssim_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_float_ms_ssim_score'],
1.0,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_score'],
76.69926875,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_score'],
99.94641666666666,
places=4)
def test_read_dataset_fps_bad_rebuf_indices(self):
train_dataset_path = VmafConfig.test_resource_path('test_dataset_fps_bad_rebufinds.py')
train_dataset = import_python_file(train_dataset_path)
with self.assertRaises(AssertionError):
train_assets = read_dataset(train_dataset)
def test_run_vmafrc_runner_yuv420p10le_sparks(self):
ref_path = VmafConfig.test_resource_path(
"yuv", "sparks_ref_480x270.yuv42010le.yuv")
dis_path = VmafConfig.test_resource_path(
"yuv", "sparks_dis_480x270.yuv42010le.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 480,
'height': 270,
'yuv_type': 'yuv420p10le'
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 480,
'height': 270,
'yuv_type': 'yuv420p10le'
})
self.runner = VmafrcQualityRunner([asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'psnr': True,
'ssim': True,
'ms_ssim': True,
})
self.runner.run(parallelize=True)
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale0_score'],
0.9240746,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale1_score'],
0.9968371999999999,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale2_score'],
0.9987575999999999,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_vif_scale3_score'],
0.9993221999999999,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_motion2_score'],
0.7523685999999999,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_adm2_score'],
0.9981770000000001,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_float_psnr_score'],
48.81622,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_float_ssim_score'],
0.99566,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_float_ms_ssim_score'],
0.9993778000000001,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale0_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale1_score'],
0.9999990000000001,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale2_score'],
0.9999990000000001,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_vif_scale3_score'],
0.9999990000000001,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_motion2_score'],
0.7523685999999999,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAFRC_float_psnr_score'],
72.0,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_float_ssim_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_float_ms_ssim_score'],
1.0,
places=4)
self.assertAlmostEqual(results[0]['VMAFRC_score'],
97.90585999999999,
places=4)
self.assertAlmostEqual(results[1]['VMAFRC_score'], 98.47138, places=4)
class BootstrapVmafQualityRunner(VmafQualityRunner):
TYPE = "BOOTSTRAP_VMAF"
VERSION = VmafQualityRunner.VERSION + '-' + 'M' + BootstrapLibsvmNusvrTrainTestModel.VERSION
ALGO_VERSION = None
# "vmaf_b_v0.6.3": plain bootstrapping, "vmaf_rb_v0.6.3": residue bootstrapping
DEFAULT_MODEL_FILEPATH = VmafConfig.model_path("vmaf_b_v0.6.3",
"vmaf_b_v0.6.3.pkl")
def _populate_result_dict(self, feature_result, pred_result):
result_dict = {}
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = pred_result[
'ys_pred'] # add quality score
result_dict[self.get_all_models_scores_key()] = pred_result[
'ys_pred_all_models'] # add quality score from all models
result_dict[self.get_bagging_scores_key()] = pred_result[
'ys_pred_bagging'] # add bagging quality score
result_dict[self.get_stddev_scores_key()] = pred_result[
'ys_pred_stddev'] # add stddev of bootstrapped quality score
result_dict[self.get_ci95_low_scores_key()] = pred_result[
'ys_pred_ci95_low'] # add ci95 of bootstrapped quality score
result_dict[self.get_ci95_high_scores_key()] = pred_result[
'ys_pred_ci95_high'] # add ci95 of bootstrapped quality score
return result_dict
@classmethod
def predict_with_model(cls, model, xs, **kwargs):
DELTA = 1e-2
result = model.predict(xs)
ys_pred_all_models = result['ys_label_pred_all_models']
ys_pred = result['ys_label_pred']
ys_pred_bagging = result['ys_label_pred_bagging']
ys_pred_stddev = result['ys_label_pred_stddev']
ys_pred_ci95_low = result['ys_label_pred_ci95_low']
ys_pred_ci95_high = result['ys_label_pred_ci95_high']
ys_pred_plus = ys_pred_bagging + DELTA
ys_pred_minus = ys_pred_bagging - DELTA
do_transform_score = cls._do_transform_score(kwargs)
if do_transform_score:
ys_pred_all_models = np.array([
cls.transform_score(model, ys_pred_some_model)
for ys_pred_some_model in ys_pred_all_models
])
ys_pred = cls.transform_score(model, ys_pred)
ys_pred_bagging = cls.transform_score(model, ys_pred_bagging)
ys_pred_plus = cls.transform_score(model, ys_pred_plus)
ys_pred_minus = cls.transform_score(model, ys_pred_minus)
ys_pred_ci95_low = cls.transform_score(model, ys_pred_ci95_low)
ys_pred_ci95_high = cls.transform_score(model, ys_pred_ci95_high)
else:
pass
if 'disable_clip_score' in kwargs and kwargs[
'disable_clip_score'] is True:
pass
else:
ys_pred_all_models = np.array([
cls.clip_score(model, ys_pred_some_model)
for ys_pred_some_model in ys_pred_all_models
])
ys_pred = cls.clip_score(model, ys_pred)
ys_pred_bagging = cls.clip_score(model, ys_pred_bagging)
ys_pred_plus = cls.clip_score(model, ys_pred_plus)
ys_pred_minus = cls.clip_score(model, ys_pred_minus)
ys_pred_ci95_low = cls.clip_score(model, ys_pred_ci95_low)
ys_pred_ci95_high = cls.clip_score(model, ys_pred_ci95_high)
# stddev score transform is applied after transform, clip, or both, or neither
slope = ((ys_pred_plus - ys_pred_minus) / (2.0 * DELTA))
ys_pred_stddev = ys_pred_stddev * slope
return {
'ys_pred_all_models': ys_pred_all_models,
'ys_pred': ys_pred,
'ys_pred_bagging': ys_pred_bagging,
'ys_pred_stddev': ys_pred_stddev,
'ys_pred_ci95_low': ys_pred_ci95_low,
'ys_pred_ci95_high': ys_pred_ci95_high,
}
def get_train_test_model_class(self):
# overide VmafQualityRunner.get_train_test_model_class
return BootstrapLibsvmNusvrTrainTestModel
@classmethod
def get_all_models_scores_key(cls):
return cls.TYPE + '_all_models_scores'
@classmethod
def get_all_models_score_key(cls):
return cls.TYPE + '_all_models_score'
@classmethod
def get_bagging_scores_key(cls):
return cls.TYPE + '_bagging_scores'
@classmethod
def get_bagging_score_key(cls):
return cls.TYPE + '_bagging_score'
@classmethod
def get_stddev_scores_key(cls):
return cls.TYPE + '_stddev_scores'
@classmethod
def get_stddev_score_key(cls):
return cls.TYPE + '_stddev_score'
@classmethod
def get_ci95_low_scores_key(cls):
return cls.TYPE + '_ci95_low_scores'
@classmethod
def get_ci95_low_score_key(cls):
return cls.TYPE + '_ci95_low_score'
@classmethod
def get_ci95_high_scores_key(cls):
return cls.TYPE + '_ci95_high_scores'
@classmethod
def get_ci95_high_score_key(cls):
return cls.TYPE + '_ci95_high_score'
def test_run_vmafrc_compare_directly_with_ossexec_420_10bit(self):
ref_path = VmafConfig.test_resource_path(
"yuv", "sparks_ref_480x270.yuv42010le.yuv")
dis_path = VmafConfig.test_resource_path(
"yuv", "sparks_dis_480x270.yuv42010le.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 480,
'height': 270,
'yuv_type': 'yuv420p10le'
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 480,
'height': 270,
'yuv_type': 'yuv420p10le'
})
self.runner = VmafrcQualityRunner([asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict={
'psnr': True,
'ssim': True,
'ms_ssim': True,
})
self.runner.run(parallelize=False)
results_rc = self.runner.results
self.runner = VmafossExecQualityRunner(
[asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
)
self.runner.run(parallelize=False)
results = self.runner.results
self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale0_score'],
results[0]['VMAFOSSEXEC_vif_scale0_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale1_score'],
results[0]['VMAFOSSEXEC_vif_scale1_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale2_score'],
results[0]['VMAFOSSEXEC_vif_scale2_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_vif_scale3_score'],
results[0]['VMAFOSSEXEC_vif_scale3_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_motion2_score'],
results[0]['VMAFOSSEXEC_motion2_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_adm2_score'],
results[0]['VMAFOSSEXEC_adm2_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_float_psnr_score'],
results[0]['VMAFOSSEXEC_psnr_score'],
places=4)
self.assertAlmostEqual(results_rc[0]['VMAFRC_float_ssim_score'],
results[0]['VMAFOSSEXEC_ssim_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_float_ms_ssim_score'],
results[0]['VMAFOSSEXEC_ms_ssim_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale0_score'],
results[1]['VMAFOSSEXEC_vif_scale0_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale1_score'],
results[1]['VMAFOSSEXEC_vif_scale1_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale2_score'],
results[1]['VMAFOSSEXEC_vif_scale2_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_vif_scale3_score'],
results[1]['VMAFOSSEXEC_vif_scale3_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_motion2_score'],
results[1]['VMAFOSSEXEC_motion2_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_adm2_score'],
results[1]['VMAFOSSEXEC_adm2_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_float_psnr_score'],
results[1]['VMAFOSSEXEC_psnr_score'],
places=4)
self.assertAlmostEqual(results_rc[1]['VMAFRC_float_ssim_score'],
results[1]['VMAFOSSEXEC_ssim_score'],
places=5)
self.assertAlmostEqual(results_rc[1]['VMAFRC_float_ms_ssim_score'],
results[1]['VMAFOSSEXEC_ms_ssim_score'],
places=5)
self.assertAlmostEqual(results_rc[0]['VMAFRC_score'],
results[0]['VMAFOSSEXEC_score'],
places=4)
self.assertAlmostEqual(results_rc[1]['VMAFRC_score'],
results[1]['VMAFOSSEXEC_score'],
places=4)
class NiqeQualityRunner(QualityRunner):
TYPE = 'NIQE'
# VERSION = '0.1'
VERSION = '0.2' # update model to py3
DEFAULT_MODEL_FILEPATH = VmafConfig.model_path('other_models',
'niqe_v0.1.pkl')
DEFAULT_FEATURE_DICT = {'NIQE_noref_feature': 'all'}
def _get_quality_scores(self, asset):
raise NotImplementedError
def _generate_result(self, asset):
raise NotImplementedError
def _get_niqe_feature_assembler_instance(self, asset):
# load TrainTestModel only to retrieve its 'feature_dict' extra info
model = self._load_model(asset)
# need this so that FeatureAssembler can find NiqeNorefFeatureExtractor:
from vmaf.core.noref_feature_extractor import NiqeNorefFeatureExtractor
feature_dict = model.get_appended_info('feature_dict')
if feature_dict is None:
feature_dict = self.DEFAULT_FEATURE_DICT
feature_optional_dict = model.get_appended_info(
'feature_optional_dict')
vmaf_fassembler = FeatureAssembler(
feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=feature_optional_dict,
optional_dict2=None,
parallelize=False,
)
return vmaf_fassembler
def _load_model(self, asset):
if self.optional_dict is not None \
and 'model_filepath' in self.optional_dict \
and self.optional_dict['model_filepath'] is not None:
model_filepath = self.optional_dict['model_filepath']
else:
model_filepath = self.DEFAULT_MODEL_FILEPATH
model = TrainTestModel.from_file(model_filepath, self.logger)
return model
def _run_on_asset(self, asset):
# Override Executor._run_on_asset(self, asset), which runs a
# FeatureAssembler, collect a feature vector, run
# TrainTestModel.predict() on it, and return a Result object
# (in this case, both Executor._run_on_asset(self, asset) and
# QualityRunner._read_result(self, asset) get bypassed.
niqe_fassembler = self._get_niqe_feature_assembler_instance(asset)
niqe_fassembler.run()
feature_result = niqe_fassembler.results[0]
# xs = NiqeTrainTestModel.get_perframe_xs_from_result(feature_result)
xs = NiqeTrainTestModel.get_xs_from_results([feature_result])
model = self._load_model(asset)
ys_pred = model.predict(xs)['ys_label_pred']
result_dict = {}
# add all feature result
result_dict.update(feature_result.result_dict)
# add quality score
result_dict[self.get_scores_key()] = ys_pred
return Result(asset, self.executor_id, result_dict)
def _remove_result(self, asset):
# Override Executor._remove_result(self, asset) by redirecting it to the
# FeatureAssembler.
vmaf_fassembler = self._get_niqe_feature_assembler_instance(asset)
vmaf_fassembler.remove_results()
__copyright__ = "Copyright 2016-2018, Netflix, Inc."
__license__ = "Apache, Version 2.0"
import matplotlib.pyplot as plt
import numpy as np
from vmaf.config import VmafConfig, DisplayConfig
from vmaf.routine import run_vmaf_cv, run_vmaf_kfold_cv
if __name__ == '__main__':
# ==== Run simple cross validation: one training and one testing dataset ====
run_vmaf_cv(
train_dataset_filepath=VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py'),
test_dataset_filepath=VmafConfig.resource_path('dataset', 'VQEGHD3_dataset.py'),
param_filepath=VmafConfig.resource_path('param', 'vmaf_v3.py'),
output_model_filepath=VmafConfig.workspace_path('model', 'test_model1.pkl'),
)
# ==== Run cross validation across genres (tough test) ====
nflx_dataset_path = VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py')
contentid_groups = [
[0, 5], # cartoon: BigBuckBunny, FoxBird
[1], # CG: BirdsInCage
[2, 6, 7], # complex: CrowdRun, OldTownCross, Seeking
[3, 4], # ElFuente: ElFuente1, ElFuente2
[8], # sports: Tennis
]
param_filepath = VmafConfig.resource_path('param', 'vmaf_v3.py')
class SpEEDMatlabFeatureExtractor(MatlabFeatureExtractor):
TYPE = 'SpEED_Matlab_feature'
VERSION = '0.1'
scale_list = [2, 3, 4]
ATOM_FEATURES = []
DERIVED_ATOM_FEATURES = []
for scale_now in scale_list:
ATOM_FEATURES.append('sspeed_' + str(scale_now))
ATOM_FEATURES.append('tspeed_' + str(scale_now))
DERIVED_ATOM_FEATURES.append('speed_' + str(scale_now))
MATLAB_WORKSPACE = VmafConfig.root_path('matlab', 'SpEED')
def _generate_result(self, asset):
# routine to call the command-line executable and generate quality
# scores in the log file.
ref_workfile_path = asset.ref_workfile_path
dis_workfile_path = asset.dis_workfile_path
log_file_path = self._get_log_file_path(asset)
current_dir = os.getcwd() + '/'
ref_workfile_path = make_absolute_path(ref_workfile_path, current_dir)
dis_workfile_path = make_absolute_path(dis_workfile_path, current_dir)
log_file_path = make_absolute_path(log_file_path, current_dir)
quality_width, quality_height = asset.quality_width_height
speed_cmd = '''{matlab} -nodisplay -nosplash -nodesktop -r "run_speed('{ref}', '{dis}', {w}, {h}, {bands}, '{yuv_type}'); exit;" >> {log_file_path}'''.format(
matlab=VmafExternalConfig.get_and_assert_matlab(),
ref=ref_workfile_path,
dis=dis_workfile_path,
w=quality_width,
h=quality_height,
bands=self.scale_list,
yuv_type=self._get_workfile_yuv_type(asset),
log_file_path=log_file_path,
)
if self.logger:
self.logger.info(speed_cmd)
os.chdir(self.MATLAB_WORKSPACE)
run_process(speed_cmd, shell=True)
os.chdir(current_dir)
@classmethod
def _post_process_result(cls, result):
# override Executor._post_process_result
def _speed(sspeed_tspeed):
sspeed, tspeed = sspeed_tspeed
if sspeed is not None and tspeed is not None:
return sspeed * tspeed
elif sspeed is None:
return tspeed
elif tspeed is None:
return sspeed
else:
return None
result = super(SpEEDMatlabFeatureExtractor,
cls)._post_process_result(result)
for scale_now in cls.scale_list:
sspeed_scale_now_scores_key = cls.get_scores_key('sspeed_' +
str(scale_now))
tspeed_scale_now_scores_key = cls.get_scores_key('tspeed_' +
str(scale_now))
speed_scale_now_scores_key = cls.get_scores_key('speed_' +
str(scale_now))
sspeed_scale_now_scores = result.result_dict[
sspeed_scale_now_scores_key]
tspeed_scale_now_scores = result.result_dict[
tspeed_scale_now_scores_key]
assert len(sspeed_scale_now_scores) == len(tspeed_scale_now_scores)
# consistent with VMAF framework, which is to multiply S and T scores per frame, then average
speed_scale_now_scores = map(
_speed, zip(sspeed_scale_now_scores, tspeed_scale_now_scores))
result.result_dict[
speed_scale_now_scores_key] = speed_scale_now_scores
# validate
for feature in cls.DERIVED_ATOM_FEATURES:
assert cls.get_scores_key(feature) in result.result_dict
return result
def test_run_testing_psnr(self):
exe = VmafConfig.root_path('run_testing')
cmd = "{exe} PSNR {dataset} --parallelize --suppress-plot".format(
exe=exe, dataset=self.dataset_filename)
ret = run_process(cmd, shell=True)
self.assertEquals(ret, 0)
from vmaf.config import VmafConfig
from vmaf.core.executor import run_executors_in_parallel
from vmaf.core.raw_extractor import DisYUVRawVideoExtractor
from vmaf.core.nn_train_test_model import ToddNoiseClassifierTrainTestModel
from vmaf.routine import read_dataset
from vmaf.tools.misc import import_python_file
# parameters
num_train = 500
num_test = 50
n_epochs = 30
seed = 0 # None
# read input dataset
dataset_path = VmafConfig.resource_path('dataset', 'BSDS500_noisy_dataset.py')
dataset = import_python_file(dataset_path)
assets = read_dataset(dataset)
# shuffle assets
np.random.seed(seed)
np.random.shuffle(assets)
assets = assets[:(num_train + num_test)]
raw_video_h5py_filepath = VmafConfig.workdir_path('rawvideo.hdf5')
raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file(
raw_video_h5py_filepath)
print '======================== Extract raw YUVs =============================='
_, raw_yuvs = run_executors_in_parallel(
def main():
if len(sys.argv) < 6:
print_usage()
return 2
try:
fmt = sys.argv[1]
width = int(sys.argv[2])
height = int(sys.argv[3])
ref_path = sys.argv[4]
dis_path = sys.argv[5]
except ValueError:
print_usage()
return 2
if width < 0 or height < 0:
print("width and height must be non-negative, but are {w} and {h}".format(w=width, h=height))
print_usage()
return 2
if fmt not in FMTS:
print_usage()
return 2
out_fmt = get_cmd_option(sys.argv, 6, len(sys.argv), '--out-fmt')
if not (out_fmt is None
or out_fmt == 'xml'
or out_fmt == 'json'
or out_fmt == 'text'):
print_usage()
return 2
pool_method = get_cmd_option(sys.argv, 6, len(sys.argv), '--pool')
if not (pool_method is None
or pool_method in POOL_METHODS):
print('--pool can only have option among {}'.format(', '.join(POOL_METHODS)))
return 2
asset = Asset(dataset="cmd", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':width, 'height':height, 'yuv_type':fmt}
)
assets = [asset]
runner_class = PsnrQualityRunner
runner = runner_class(
assets, None, fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
# run
runner.run()
result = runner.results[0]
# pooling
if pool_method == 'harmonic_mean':
result.set_score_aggregate_method(ListStats.harmonic_mean)
elif pool_method == 'min':
result.set_score_aggregate_method(np.min)
elif pool_method == 'median':
result.set_score_aggregate_method(np.median)
elif pool_method == 'perc5':
result.set_score_aggregate_method(ListStats.perc5)
elif pool_method == 'perc10':
result.set_score_aggregate_method(ListStats.perc10)
elif pool_method == 'perc20':
result.set_score_aggregate_method(ListStats.perc20)
else: # None or 'mean'
pass
# output
if out_fmt == 'xml':
print(result.to_xml())
elif out_fmt == 'json':
print(result.to_json())
else: # None or 'text'
print(str(result))
return 0
def test_run_vmaf_fextractor_with_resampling(self):
ref_path = VmafConfig.test_resource_path("yuv",
"src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"src01_hrc01_576x324.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324,
'quality_width': 160,
'quality_height': 90
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=2,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324,
'quality_width': 160,
'quality_height': 90
})
self.fextractor = VmafFeatureExtractor([asset, asset_original],
None,
fifo_mode=False)
self.fextractor.run(parallelize=False)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.74165043750000004,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
1.4066421666666666,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'],
0.9807496875,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
27.319241250000001,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
1.4066421666666666,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.682829895833333,
places=4)
def test_run_strred_fextractor_blackframes(self):
ref_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"flat_1920_1080_10.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
from vmaf.core.result_store import FileSystemResultStore
result_store = FileSystemResultStore(logger=None)
self.fextractor = StrredFeatureExtractor([asset, asset_original],
None,
fifo_mode=True,
result_store=result_store)
self.fextractor.run(parallelize=True)
result0, result1 = self.fextractor.results
import os
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result1)))
self.fextractor.run(parallelize=True)
results = self.fextractor.results
# ignore NaN
for result in results:
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'],
1220.5679849999999,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'],
50983.3097155,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'],
62228595.6081,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'],
0.0,
places=4)
