def test_run_parallel_moment_noref_fextractor(self):
print 'test on running Moment noref feature extractor on NorefAssets in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = NorefAsset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = NorefAsset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
MomentNorefFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['Moment_noref_feature_1st_score'], 61.332006624999984)
self.assertAlmostEqual(results[0]['Moment_noref_feature_2nd_score'], 4798.659574041666)
self.assertAlmostEqual(results[0]['Moment_noref_feature_var_score'], 1036.8371843488285)
self.assertAlmostEqual(results[1]['Moment_noref_feature_1st_score'], 59.788567297525134)
self.assertAlmostEqual(results[1]['Moment_noref_feature_2nd_score'], 4696.668388042271)
self.assertAlmostEqual(results[1]['Moment_noref_feature_var_score'], 1121.519917231207)
def test_run_parallel_moment_fextractor(self):
print 'test on running Moment feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.fextractors, results = run_executors_in_parallel(
MomentFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['Moment_feature_ref1st_score'],
59.788567297525134)
self.assertAlmostEqual(results[0]['Moment_feature_ref2nd_score'],
4696.668388042269)
self.assertAlmostEqual(results[0]['Moment_feature_refvar_score'],
1121.519917231203)
self.assertAlmostEqual(results[0]['Moment_feature_dis1st_score'],
61.332006624999984)
self.assertAlmostEqual(results[0]['Moment_feature_dis2nd_score'],
4798.659574041666)
self.assertAlmostEqual(results[0]['Moment_feature_disvar_score'],
1036.837184348847)
self.assertAlmostEqual(results[1]['Moment_feature_ref1st_score'],
59.788567297525134)
self.assertAlmostEqual(results[1]['Moment_feature_ref2nd_score'],
4696.668388042269)
self.assertAlmostEqual(results[1]['Moment_feature_refvar_score'],
1121.519917231203)
self.assertAlmostEqual(results[1]['Moment_feature_dis1st_score'],
59.788567297525134)
self.assertAlmostEqual(results[1]['Moment_feature_dis2nd_score'],
4696.668388042269)
self.assertAlmostEqual(results[1]['Moment_feature_disvar_score'],
1121.519917231203)
def test_run_parallel_ssim_fextractor(self):
print 'test on running SSIM feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
SsimFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_score'], 0.857692208333, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_l_score'], 0.9923925625, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_c_score'], 0.9611274375, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_s_score'], 0.897844791667, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_l_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_c_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_s_score'], 1.0, places=4)
def test_run_parallel_psnr_runner(self):
print 'test on running PSNR quality runner in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.runners, results = run_executors_in_parallel(
PsnrQualityRunner,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None
)
self.assertAlmostEqual(results[0]['PSNR_score'], 30.755063979166664)
self.assertAlmostEqual(results[1]['PSNR_score'], 60.0)
def test_run_parallel_ssim_fextractor(self):
print 'test on running SSIM feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
SsimFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_score'], 0.86325137500000004, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_l_score'], 0.99814749999999997, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_c_score'], 0.96132239583333323, places=4)
self.assertAlmostEqual(results[0]['SSIM_feature_ssim_s_score'], 0.89770760416666662, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_l_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_c_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['SSIM_feature_ssim_s_score'], 1.0, places=4)
def setUp(self):
train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_noisy.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
self.raw_video_h5py_filepath = config.ROOT + '/workspace/workdir/rawvideo.hdf5'
self.raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file(
self.raw_video_h5py_filepath)
optional_dict2 = {'h5py_file': self.raw_video_h5py_file}
_, self.features = run_executors_in_parallel(
DisYUVRawVideoExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=
False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor
result_store=None,
optional_dict=None,
optional_dict2=optional_dict2,
)
np.random.seed(0)
np.random.shuffle(self.features)
self.patch_h5py_filepath = config.ROOT + '/workspace/workdir/patch.hdf5'
self.model_filename = config.ROOT + "/workspace/model/test_save_load.pkl"
NeuralNetTrainTestModel.reset()
def test_run_parallel_psnr_runner(self):
print 'test on running PSNR quality runner in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.runners, results = run_executors_in_parallel(
PsnrQualityRunner, [asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None)
self.assertAlmostEqual(results[0]['PSNR_score'], 30.755063979166664)
self.assertAlmostEqual(results[1]['PSNR_score'], 60.0)
def test_run_parallel_ms_ssim_fextractor(self):
print 'test on running MS-SSIM feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
MsSsimFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.96324620833333319, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9981474999999999, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.9613223958333336, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.8977076041666665, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.9989961250000002, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9858215416666668, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.9411672708333335, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.9992356458333332, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.9970406458333333, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.9779967291666667, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.9992921041666665, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.9995884375000003, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.9938731666666668, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.99940356249999995, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.99990762500000008, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.99822306250000004, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_parallel_ms_ssim_fextractor(self):
print 'test on running MS-SSIM feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
MsSsimFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.9632498125, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9923925625, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.9611274375, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.897844791667, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.9954706875, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9857694375, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.941185875, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.997437458333, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.997034020833, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.977992145833, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.998035583333, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.999588104167, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.99387125, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.9995495, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.999907625, places=4)
self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.998222583333, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4)
self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_parallel_vamf_fextractor(self):
print 'test on running VMAF feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499, places=4)
def test_run_parallel_dis_y_fextractor(self):
print 'test on running dis YUV raw video extractor in parallel (disabled)...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=2,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath)
optional_dict2 = {'h5py_file': h5py_file}
self.fextractors, results = run_executors_in_parallel(
DisYUVRawVideoExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=False, # Can't run parallel: can't pickle FileID objects
result_store=None,
optional_dict={'channels': 'yu'},
optional_dict2=optional_dict2)
self.assertAlmostEqual(np.mean(results[0]['dis_y']),
61.332006579182384,
places=4)
self.assertAlmostEquals(np.mean(results[1]['dis_y']),
59.788567297525148,
places=4)
self.assertAlmostEqual(np.mean(results[0]['dis_u']),
115.23227407335962,
places=4)
self.assertAlmostEquals(np.mean(results[1]['dis_u']),
114.49701717535437,
places=4)
with self.assertRaises(KeyError):
np.mean(results[0]['dis_v'])
DisYUVRawVideoExtractor.close_h5py_file(h5py_file)
def test_run_parallel_vamf_fextractor(self):
print 'test on running VMAF feature extractor in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499, places=4)
def test_explain_train_test_model(self):
model_class = SklearnRandomForestTrainTestModel
train_dataset_path = config.ROOT + '/python/test/resource/' \
'test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
_, self.features = run_executors_in_parallel(
MomentNorefFeatureExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
xys = model_class.get_xys_from_results(self.features[:7])
model = model_class({'norm_type':'normalize', 'random_state':0}, None)
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'],
['Moment_noref_feature_1st_score',
'Moment_noref_feature_2nd_score',
'Moment_noref_feature_var_score']
)
def test_explain_train_test_model(self):
model_class = SklearnRandomForestTrainTestModel
train_dataset_path = config.ROOT + '/python/test/resource/' \
'test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
_, self.features = run_executors_in_parallel(
MomentNorefFeatureExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
xys = model_class.get_xys_from_results(self.features[:7])
model = model_class({'norm_type':'normalize', 'random_state':0}, None)
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'],
['Moment_noref_feature_1st_score',
'Moment_noref_feature_2nd_score',
'Moment_noref_feature_var_score']
)
def test_run_parallel_vamf_runner_with_model(self):
print 'test on running VMAF quality runner in parallel with custom model...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.runners, results = run_executors_in_parallel(
VmafQualityRunner, [asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict={
'model_filepath':
config.ROOT + "/resource/model/nflx_vmaff_rf_v1.pkl",
})
self.assertAlmostEqual(results[0]['VMAF_score'], 73.79861111111113)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'],
0.9155242291666666)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_score'], 98.22048611111109)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.030914145833322)
def test_run_parallel_vmaf_legacy_runner(self):
print 'test on running VMAF (legacy) quality runner in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.runners, results = run_executors_in_parallel(
VmafLegacyQualityRunner, [asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None)
self.assertAlmostEqual(results[0]['VMAF_legacy_score'],
60.27316952679754)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'],
0.9155242291666666)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_legacy_score'],
95.65756240092573)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.030914145833322)
def setUp(self):
train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
_, self.features = run_executors_in_parallel(
MomentNorefFeatureExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
def setUp(self):
train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
_, self.features = run_executors_in_parallel(
MomentNorefFeatureExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
def test_run_parallel_moment_fextractor(self):
print "test on running Moment feature extractor in parallel..."
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(
dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={"width": 576, "height": 324},
)
asset_original = Asset(
dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={"width": 576, "height": 324},
)
self.fextractors, results = run_executors_in_parallel(
MomentFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]["Moment_feature_ref1st_score"], 59.788567297525134, places=4)
self.assertAlmostEqual(results[0]["Moment_feature_ref2nd_score"], 4696.668388042269, places=4)
self.assertAlmostEqual(results[0]["Moment_feature_refvar_score"], 1121.519917231203, places=4)
self.assertAlmostEqual(results[0]["Moment_feature_dis1st_score"], 61.332006624999984, places=4)
self.assertAlmostEqual(results[0]["Moment_feature_dis2nd_score"], 4798.659574041666, places=4)
self.assertAlmostEqual(results[0]["Moment_feature_disvar_score"], 1036.837184348847, places=4)
self.assertAlmostEqual(results[1]["Moment_feature_ref1st_score"], 59.788567297525134, places=4)
self.assertAlmostEqual(results[1]["Moment_feature_ref2nd_score"], 4696.668388042269, places=4)
self.assertAlmostEqual(results[1]["Moment_feature_refvar_score"], 1121.519917231203, places=4)
self.assertAlmostEqual(results[1]["Moment_feature_dis1st_score"], 59.788567297525134, places=4)
self.assertAlmostEqual(results[1]["Moment_feature_dis2nd_score"], 4696.668388042269, places=4)
self.assertAlmostEqual(results[1]["Moment_feature_disvar_score"], 1121.519917231203, places=4)
def run(self):
"""
Do all the calculation here.
:return:
"""
# for each FeatureExtractor_type key in feature_dict, find the subclass
# of FeatureExtractor, run, and put results in a dict
for fextractor_type in self.feature_dict:
# fextractor = self._get_fextractor_instance(fextractor_type)
# fextractor.run()
# results = fextractor.results
fextractor_class = FeatureExtractor.find_subclass(fextractor_type)
_, results = run_executors_in_parallel(
fextractor_class,
assets=self.assets,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
parallelize=self.parallelize,
result_store=self.result_store,
optional_dict=self.optional_dict,
optional_dict2=self.optional_dict2,
)
self.type2results_dict[fextractor_type] = results
# assemble an output dict with demanded atom features
# atom_features_dict = self.fextractor_atom_features_dict
result_dicts = [dict() for _ in self.assets]
for fextractor_type in self.feature_dict:
assert fextractor_type in self.type2results_dict
for atom_feature in self._get_atom_features(fextractor_type):
scores_key = self._get_scores_key(fextractor_type,
atom_feature)
for result_index, result in enumerate(
self.type2results_dict[fextractor_type]):
result_dicts[result_index][scores_key] = result[scores_key]
self.results = map(
lambda (asset, result_dict): BasicResult(asset, result_dict),
zip(self.assets, result_dicts))
def setUp(self):
train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
self.h5py_filepath = config.ROOT + '/workspace/workdir/test.hdf5'
self.h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath)
optional_dict2 = {'h5py_file': self.h5py_file}
_, self.features = run_executors_in_parallel(
DisYUVRawVideoExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor
result_store=None,
optional_dict=None,
optional_dict2=optional_dict2,
)
def setUp(self):
train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
self.h5py_filepath = config.ROOT + '/workspace/workdir/test.hdf5'
self.h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath)
optional_dict2 = {'h5py_file': self.h5py_file}
_, self.features = run_executors_in_parallel(
DisYUVRawVideoExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor
result_store=None,
optional_dict=None,
optional_dict2=optional_dict2,
)
def run(self):
"""
Do all the calculation here.
:return:
"""
# for each FeatureExtractor_type key in feature_dict, find the subclass
# of FeatureExtractor, run, and put results in a dict
for fextractor_type in self.feature_dict:
# fextractor = self._get_fextractor_instance(fextractor_type)
# fextractor.run()
# results = fextractor.results
fextractor_class = FeatureExtractor.find_subclass(fextractor_type)
_, results = run_executors_in_parallel(
fextractor_class,
assets=self.assets,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
parallelize=self.parallelize,
result_store=self.result_store,
optional_dict=self.optional_dict,
optional_dict2=self.optional_dict2,
)
self.type2results_dict[fextractor_type] = results
# assemble an output dict with demanded atom features
# atom_features_dict = self.fextractor_atom_features_dict
result_dicts = [dict() for _ in self.assets]
for fextractor_type in self.feature_dict:
assert fextractor_type in self.type2results_dict
for atom_feature in self._get_atom_features(fextractor_type):
scores_key = self._get_scores_key(fextractor_type, atom_feature)
for result_index, result in enumerate(self.type2results_dict[
fextractor_type]):
result_dicts[result_index][scores_key] = result[scores_key]
self.results = map(
lambda (asset, result_dict): BasicResult(asset, result_dict),
zip(self.assets, result_dicts)
)
def test_run_parallel_ssim_fextractor(self):
print "test on running SSIM feature extractor in parallel..."
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(
dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={"width": 576, "height": 324},
)
asset_original = Asset(
dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={"width": 576, "height": 324},
)
self.fextractors, results = run_executors_in_parallel(
SsimFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]["SSIM_feature_ssim_score"], 0.86322654166666657, places=4)
self.assertAlmostEqual(results[0]["SSIM_feature_ssim_l_score"], 0.9981474583333334, places=4)
self.assertAlmostEqual(results[0]["SSIM_feature_ssim_c_score"], 0.96126793750000006, places=4)
self.assertAlmostEqual(results[0]["SSIM_feature_ssim_s_score"], 0.89773633333333336, places=4)
self.assertAlmostEqual(results[1]["SSIM_feature_ssim_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["SSIM_feature_ssim_l_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["SSIM_feature_ssim_c_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["SSIM_feature_ssim_s_score"], 1.0, places=4)
def test_run_parallel_dis_y_fextractor(self):
print 'test on running dis YUV raw video extractor in parallel (disabled)...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=2,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath)
optional_dict2 = {'h5py_file': h5py_file}
self.fextractors, results = run_executors_in_parallel(
DisYUVRawVideoExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=False, # Can't run parallel: can't pickle FileID objects
result_store=None,
optional_dict={'channels': 'yu'},
optional_dict2=optional_dict2
)
self.assertAlmostEqual(np.mean(results[0]['dis_y']), 61.332006579182384, places=4)
self.assertAlmostEquals(np.mean(results[1]['dis_y']), 59.788567297525148, places=4)
self.assertAlmostEqual(np.mean(results[0]['dis_u']), 115.23227407335962, places=4)
self.assertAlmostEquals(np.mean(results[1]['dis_u']), 114.49701717535437, places=4)
with self.assertRaises(KeyError):
np.mean(results[0]['dis_v'])
DisYUVRawVideoExtractor.close_h5py_file(h5py_file)
def test_run_parallel_vamf_runner_with_model(self):
print 'test on running VMAF quality runner in parallel with custom model...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.runners, results = run_executors_in_parallel(
VmafQualityRunner,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict={
'model_filepath':config.ROOT + "/resource/model/nflx_vmaff_rf_v1.pkl",
}
)
self.assertAlmostEqual(results[0]['VMAF_score'], 73.79861111111113)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_score'], 98.22048611111109)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def test_run_parallel_vmaf_legacy_runner(self):
print 'test on running VMAF (legacy) quality runner in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.runners, results = run_executors_in_parallel(
VmafLegacyQualityRunner,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None
)
self.assertAlmostEqual(results[0]['VMAF_legacy_score'], 60.27316952679754)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_legacy_score'], 95.65756240092573)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def main():
if len(sys.argv) < 2:
print_usage()
return 2
input_filepath = sys.argv[1]
model_path = get_cmd_option(sys.argv, 2, len(sys.argv), '--model')
out_fmt = get_cmd_option(sys.argv, 2, len(sys.argv), '--out-fmt')
if not (out_fmt is None or out_fmt == 'xml' or out_fmt == 'json'
or out_fmt == 'text'):
print_usage()
return 2
pool_method = get_cmd_option(sys.argv, 2, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print_usage()
return 2
parallelize = cmd_option_exists(sys.argv, 2, len(sys.argv),
'--parallelize')
assets = []
line_idx = 0
with open(input_filepath, "rt") as input_file:
for line in input_file.readlines():
# match comment
mo = re.match(r"^#", line)
if mo:
print "Skip commented line: {}".format(line)
continue
# match whitespace
mo = re.match(r"[\s]+", line)
if mo:
continue
# example: yuv420p 576 324 ref.yuv dis.yuv
mo = re.match(r"([\S]+) ([0-9]+) ([0-9]+) ([\S]+) ([\S]+)", line)
if not mo or mo.group(1) not in FMTS:
print "Unknown format: {}".format(line)
print_usage()
return 1
fmt = mo.group(1)
width = int(mo.group(2))
height = int(mo.group(3))
ref_file = mo.group(4)
dis_file = mo.group(5)
asset = Asset(dataset="cmd",
content_id=0,
asset_id=line_idx,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_file,
dis_path=dis_file,
asset_dict={
'width': width,
'height': height,
'yuv_type': fmt
})
assets.append(asset)
line_idx += 1
runner_class = VmafQualityRunner
if model_path is None:
optional_dict = None
else:
optional_dict = {'model_filepath': model_path}
# construct an VmafQualityRunner object to assert assets, and to remove
_ = runner_class(
assets,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=optional_dict,
optional_dict2=None,
)
runners, results = run_executors_in_parallel(
runner_class,
assets,
fifo_mode=True,
delete_workdir=True,
parallelize=parallelize,
result_store=None,
optional_dict=optional_dict,
optional_dict2=None,
)
# output
for result in results:
# pooling
if pool_method == 'harmonic_mean':
result.set_score_aggregate_method(ListStats.harmonic_mean)
if pool_method == 'min':
result.set_score_aggregate_method(np.min)
else: # None or 'mean'
pass
if out_fmt == 'xml':
print result.to_xml()
elif out_fmt == 'json':
print result.to_json()
else: # None or 'json'
print '============================'
print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id)
print '============================'
print str(result)
return 0
# shuffle assets
np.random.seed(seed)
np.random.shuffle(assets)
assets = assets[:(num_train + num_test)]
raw_video_h5py_filepath = config.ROOT + '/workspace/workdir/rawvideo.hdf5'
raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file(raw_video_h5py_filepath)
print '======================== Extract raw YUVs =============================='
_, raw_yuvs = run_executors_in_parallel(
DisYUVRawVideoExtractor,
assets,
fifo_mode=True,
delete_workdir=True,
parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor
result_store=None,
optional_dict=None,
optional_dict2={'h5py_file': raw_video_h5py_file})
patch_h5py_filepath = config.ROOT + '/workspace/workdir/patch.hdf5'
patch_h5py_file = ToddNoiseClassifierTrainTestModel.open_h5py_file(patch_h5py_filepath)
model = ToddNoiseClassifierTrainTestModel(
param_dict={
'seed': seed,
'n_epochs': n_epochs,
},
logger=None,
optional_dict2={ # for options that won't impact the result
# 'checkpoints_dir': config.ROOT + '/workspace/checkpoints_dir',
def test_on_dataset(test_dataset,
runner_class,
ax,
result_store,
model_filepath,
parallelize=True,
fifo_mode=True,
aggregate_method=np.mean):
test_assets = read_dataset(test_dataset)
optional_dict = {'model_filepath': model_filepath}
# construct an quality runner object to assert assets only
runner = runner_class(
test_assets,
None,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=optional_dict,
)
try:
# run
_, results = run_executors_in_parallel(
runner_class,
test_assets,
fifo_mode=fifo_mode,
delete_workdir=True,
parallelize=parallelize,
result_store=result_store,
optional_dict=optional_dict,
)
for result in results:
result.set_aggregate_method(aggregate_method)
# plot
groundtruths = map(lambda asset: asset.groundtruth, test_assets)
predictions = map(lambda result: result[runner_class.get_score_key()],
results)
stats = TrainTestModel.get_stats(groundtruths, predictions)
print 'Stats on testing data: {}'.format(
TrainTestModel.format_stats(stats))
if ax is not None:
content_ids = map(lambda asset: asset.content_id, test_assets)
TrainTestModel.plot_scatter(ax, stats, content_ids)
ax.set_xlabel('DMOS')
ax.set_ylabel("Predicted Score")
ax.grid()
# ax.set_title( "Dataset: {dataset}, Runner: {runner}\n{stats}".format(
ax.set_title("{runner}\n{stats}".format(
dataset=test_assets[0].dataset,
# runner=results[0].executor_id,
runner=runner_class.TYPE,
stats=TrainTestModel.format_stats(stats),
# stats="",
# stats=TrainTestModel.format_stats3(stats),
))
except Exception as e:
print "Error: " + str(e)
return test_assets, results
def main():
if len(sys.argv) < 2:
print_usage()
return 2
input_filepath = sys.argv[1]
model_path = get_cmd_option(sys.argv, 2, len(sys.argv), '--model')
out_fmt = get_cmd_option(sys.argv, 2, len(sys.argv), '--out-fmt')
if not (out_fmt is None
or out_fmt == 'xml'
or out_fmt == 'json'
or out_fmt == 'text'):
print_usage()
return 2
pool_method = get_cmd_option(sys.argv, 2, len(sys.argv), '--pool')
if not (pool_method is None
or pool_method in POOL_METHODS):
print '--pool can only have option among {}'.format(', '.join(POOL_METHODS))
return 2
parallelize = cmd_option_exists(sys.argv, 2, len(sys.argv), '--parallelize')
assets = []
line_idx = 0
with open(input_filepath, "rt") as input_file:
for line in input_file.readlines():
# match comment
mo = re.match(r"^#", line)
if mo:
print "Skip commented line: {}".format(line)
continue
# match whitespace
mo = re.match(r"[\s]+", line)
if mo:
continue
# example: yuv420p 576 324 ref.yuv dis.yuv
mo = re.match(r"([\S]+) ([0-9]+) ([0-9]+) ([\S]+) ([\S]+)", line)
if not mo or mo.group(1) not in FMTS:
print "Unknown format: {}".format(line)
print_usage()
return 1
fmt = mo.group(1)
width = int(mo.group(2))
height = int(mo.group(3))
ref_file = mo.group(4)
dis_file = mo.group(5)
asset = Asset(dataset="cmd",
content_id=0,
asset_id=line_idx,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_file,
dis_path=dis_file,
asset_dict={'width':width, 'height':height, 'yuv_type':fmt}
)
assets.append(asset)
line_idx += 1
runner_class = VmafQualityRunner
if model_path is None:
optional_dict = None
else:
optional_dict = {'model_filepath':model_path}
# construct an VmafQualityRunner object to assert assets, and to remove
_ = runner_class(assets,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=optional_dict,
optional_dict2=None,
)
runners, results = run_executors_in_parallel(
runner_class,
assets,
fifo_mode=True,
delete_workdir=True,
parallelize=parallelize,
result_store=None,
optional_dict=optional_dict,
optional_dict2=None,
)
# output
for result in results:
# pooling
if pool_method == 'harmonic_mean':
result.set_score_aggregate_method(ListStats.harmonic_mean)
elif pool_method == 'min':
result.set_score_aggregate_method(np.min)
elif pool_method == 'median':
result.set_score_aggregate_method(np.median)
elif pool_method == 'perc5':
result.set_score_aggregate_method(ListStats.perc5)
elif pool_method == 'perc10':
result.set_score_aggregate_method(ListStats.perc10)
elif pool_method == 'perc20':
result.set_score_aggregate_method(ListStats.perc20)
else: # None or 'mean'
pass
if out_fmt == 'xml':
print result.to_xml()
elif out_fmt == 'json':
print result.to_json()
else: # None or 'json'
print '============================'
print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id)
print '============================'
print str(result)
return 0
runner = runner_class(
assets,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=optional_dict,
)
try:
# run
runners, results = run_executors_in_parallel(
runner_class,
assets,
fifo_mode=True,
delete_workdir=True,
parallelize=parallelize,
result_store=None,
optional_dict=optional_dict,
)
# output
for result in results:
print '============================'
print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id)
print '============================'
print str(result)
finally:
pass
def test_on_dataset(
test_dataset,
runner_class,
ax,
result_store,
model_filepath,
parallelize=True,
fifo_mode=True,
aggregate_method=np.mean,
type="regressor",
**kwargs
):
if type == "regressor":
model_type = RegressorMixin
elif type == "classifier":
model_type = ClassifierMixin
else:
assert False
test_assets = read_dataset(test_dataset, **kwargs)
test_raw_assets = None
try:
for test_asset in test_assets:
assert test_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
subj_model_class = (
kwargs["subj_model_class"]
if "subj_model_class" in kwargs and kwargs["subj_model_class"] is not None
else DmosModel
)
subjective_model = subj_model_class(RawDatasetReader(test_dataset))
subjective_model.run_modeling(**kwargs)
test_dataset_aggregate = subjective_model.to_aggregated_dataset(**kwargs)
test_raw_assets = test_assets
test_assets = read_dataset(test_dataset_aggregate, **kwargs)
if model_filepath is not None:
optional_dict = {"model_filepath": model_filepath}
else:
optional_dict = None
# construct an quality runner object to assert assets only
runner = runner_class(
test_assets,
None,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
# run
_, results = run_executors_in_parallel(
runner_class,
test_assets,
fifo_mode=fifo_mode,
delete_workdir=True,
parallelize=parallelize,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
for result in results:
result.set_score_aggregate_method(aggregate_method)
# plot
groundtruths = map(lambda asset: asset.groundtruth, test_assets)
predictions = map(lambda result: result[runner_class.get_score_key()], results)
raw_grountruths = None if test_raw_assets is None else map(lambda asset: asset.raw_groundtruth, test_raw_assets)
stats = model_type.get_stats(groundtruths, predictions, ys_label_raw=raw_grountruths)
print "Stats on testing data: {}".format(model_type.format_stats(stats))
if ax is not None:
content_ids = map(lambda asset: asset.content_id, test_assets)
model_type.plot_scatter(ax, stats, content_ids)
ax.set_xlabel("True Score")
ax.set_ylabel("Predicted Score")
ax.grid()
ax.set_title(
"{runner}\n{stats}".format(
dataset=test_assets[0].dataset, runner=runner_class.TYPE, stats=model_type.format_stats(stats)
)
)
return test_assets, results
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9254334398006141)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
22.533456770833329)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'],
644527.3311971038)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'],
1449635.3812459996)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'],
6899.815530270836)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'],
7535.801140312499)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'],
34.15266368750002)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'],
0.3655846219305399)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'],
0.7722301581694561)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'],
0.8681486658208089)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'],
0.9207121810522212)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.030914145833322)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'],
1449635.3522745417)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'],
1449635.3812459996)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'],
7535.801140312499)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'],
7535.801140312499)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'],
41.65012097916668)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'],
1.0000000132944864)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'],
0.9999998271651448)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'],
0.9999998649680067)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'],
0.9999998102499)
def test_on_dataset(test_dataset,
runner_class,
ax,
result_store,
model_filepath,
parallelize=True,
fifo_mode=True,
aggregate_method=np.mean,
type='regressor',
**kwargs):
if type == 'regressor':
model_type = RegressorMixin
elif type == 'classifier':
model_type = ClassifierMixin
else:
assert False
test_assets = read_dataset(test_dataset, **kwargs)
if model_filepath is not None:
optional_dict = {'model_filepath': model_filepath}
else:
optional_dict = None
# construct an quality runner object to assert assets only
runner = runner_class(
test_assets,
None,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
try:
# run
_, results = run_executors_in_parallel(
runner_class,
test_assets,
fifo_mode=fifo_mode,
delete_workdir=True,
parallelize=parallelize,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
for result in results:
result.set_score_aggregate_method(aggregate_method)
# plot
groundtruths = map(lambda asset: asset.groundtruth, test_assets)
predictions = map(lambda result: result[runner_class.get_score_key()],
results)
stats = model_type.get_stats(groundtruths, predictions)
print 'Stats on testing data: {}'.format(
model_type.format_stats(stats))
if ax is not None:
content_ids = map(lambda asset: asset.content_id, test_assets)
model_type.plot_scatter(ax, stats, content_ids)
ax.set_xlabel('True Score')
ax.set_ylabel("Predicted Score")
ax.grid()
ax.set_title("{runner}\n{stats}".format(
dataset=test_assets[0].dataset,
runner=runner_class.TYPE,
stats=model_type.format_stats(stats),
))
except Exception as e:
print "Error: " + str(e)
raise e
return test_assets, results
def test_run_parallel_vamf_fextractor_with_result_store(self):
print "test on running VMAF feature extractor with result store " "in parallel..."
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(
dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={"width": 576, "height": 324},
)
asset_original = Asset(
dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={"width": 576, "height": 324},
)
result_store = FileSystemResultStore(logger=None)
print " running for the first time with fresh calculation..."
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print " running for the second time with stored results..."
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_score"], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_motion_score"], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm2_score"], 0.925421075027, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_ansnr_score"], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_num_score"], 712650.023478, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_den_score"], 1597314.95249, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_num_score"], 6899.24648475, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_den_score"], 7535.29963308, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_anpsnr_score"], 34.164776875, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale0_score"], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale1_score"], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale2_score"], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale3_score"], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale0_score"], 0.86711114907261122, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale1_score"], 0.87407085185224942, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale2_score"], 0.92448300466700284, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale3_score"], 0.96367203202196239, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif2_score"], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm3_score"], 0.90733425940345658, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_motion_score"], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_ansnr_score"], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_num_score"], 1597314.86733, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_den_score"], 1597314.95249, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_num_score"], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_den_score"], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_anpsnr_score"], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm3_score"], 1.0, places=4)
def test_run_parallel_ms_ssim_fextractor(self):
print "test on running MS-SSIM feature extractor in parallel..."
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(
dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={"width": 576, "height": 324},
)
asset_original = Asset(
dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={"width": 576, "height": 324},
)
self.fextractors, results = run_executors_in_parallel(
MsSsimFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_score"], 0.9632498125, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale0_score"], 0.9981474583333334, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale0_score"], 0.96126793750000006, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale0_score"], 0.89773633333333336, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale1_score"], 0.99899612500000001, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale1_score"], 0.9857694375, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale1_score"], 0.941185875, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale2_score"], 0.99923564583333324, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale2_score"], 0.997034020833, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale2_score"], 0.977992145833, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale3_score"], 0.99929210416666658, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale3_score"], 0.999588104167, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale3_score"], 0.99387125, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale4_score"], 0.99940356249999995, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale4_score"], 0.999907625, places=4)
self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale4_score"], 0.998222583333, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale4_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale4_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale4_score"], 1.0, places=4)
def test_on_dataset(test_dataset, runner_class, ax,
result_store, model_filepath,
parallelize=True, fifo_mode=True,
aggregate_method=np.mean):
test_assets = read_dataset(test_dataset)
optional_dict = {
'model_filepath':model_filepath
}
# construct an quality runner object to assert assets only
runner = runner_class(test_assets,
None,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=optional_dict,
)
try:
# run
_, results = run_executors_in_parallel(
runner_class,
test_assets,
fifo_mode=fifo_mode,
delete_workdir=True,
parallelize=parallelize,
result_store=result_store,
optional_dict=optional_dict,
)
for result in results:
result.set_aggregate_method(aggregate_method)
# plot
groundtruths = map(lambda asset: asset.groundtruth, test_assets)
predictions = map(lambda result: result[runner_class.get_score_key()], results)
stats = TrainTestModel.get_stats(groundtruths, predictions)
print 'Stats on testing data: {}'.format(TrainTestModel.format_stats(stats))
if ax is not None:
content_ids = map(lambda asset: asset.content_id, test_assets)
TrainTestModel.plot_scatter(ax, stats, content_ids)
ax.set_xlabel('DMOS')
ax.set_ylabel("Predicted Score")
ax.grid()
# ax.set_title( "Dataset: {dataset}, Runner: {runner}\n{stats}".format(
ax.set_title( "{runner}\n{stats}".format(
dataset=test_assets[0].dataset,
# runner=results[0].executor_id,
runner=runner_class.TYPE,
stats=TrainTestModel.format_stats(stats),
# stats="",
# stats=TrainTestModel.format_stats3(stats),
))
except Exception as e:
print "Error: " + str(e)
return test_assets, results
# construct an VmafQualityRunner object to assert assets, and to remove
runner = runner_class(assets,
None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=optional_dict,
)
try:
# run
runners, results = run_executors_in_parallel(
runner_class,
assets,
fifo_mode=True,
delete_workdir=True,
parallelize=parallelize,
result_store=None,
optional_dict=optional_dict,
)
# output
for result in results:
print '============================'
print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id)
print '============================'
print str(result)
finally:
pass
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.925421075027, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.24648475, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.29963308, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.86711114907261122, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.87407085185224942, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.92448300466700284, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96367203202196239, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.90733425940345658, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
def test_on_dataset(test_dataset,
runner_class,
ax,
result_store,
model_filepath,
parallelize=True,
fifo_mode=True,
aggregate_method=np.mean,
type='regressor',
**kwargs):
if type == 'regressor':
model_type = RegressorMixin
elif type == 'classifier':
model_type = ClassifierMixin
else:
assert False
test_assets = read_dataset(test_dataset, **kwargs)
test_raw_assets = None
try:
for test_asset in test_assets:
assert test_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
subj_model_class = kwargs[
'subj_model_class'] if 'subj_model_class' in kwargs and kwargs[
'subj_model_class'] is not None else DmosModel
subjective_model = subj_model_class(RawDatasetReader(test_dataset))
subjective_model.run_modeling(**kwargs)
test_dataset_aggregate = subjective_model.to_aggregated_dataset(
**kwargs)
test_raw_assets = test_assets
test_assets = read_dataset(test_dataset_aggregate, **kwargs)
if model_filepath is not None:
optional_dict = {'model_filepath': model_filepath}
else:
optional_dict = None
# construct an quality runner object to assert assets only
runner = runner_class(
test_assets,
None,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
# run
_, results = run_executors_in_parallel(
runner_class,
test_assets,
fifo_mode=fifo_mode,
delete_workdir=True,
parallelize=parallelize,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
for result in results:
result.set_score_aggregate_method(aggregate_method)
# plot
groundtruths = map(lambda asset: asset.groundtruth, test_assets)
predictions = map(lambda result: result[runner_class.get_score_key()],
results)
raw_grountruths = None if test_raw_assets is None else \
map(lambda asset: asset.raw_groundtruth, test_raw_assets)
stats = model_type.get_stats(groundtruths,
predictions,
ys_label_raw=raw_grountruths)
print 'Stats on testing data: {}'.format(model_type.format_stats(stats))
if ax is not None:
content_ids = map(lambda asset: asset.content_id, test_assets)
model_type.plot_scatter(ax, stats, content_ids)
ax.set_xlabel('True Score')
ax.set_ylabel("Predicted Score")
ax.grid()
ax.set_title("{runner}\n{stats}".format(
dataset=test_assets[0].dataset,
runner=runner_class.TYPE,
stats=model_type.format_stats(stats),
))
return test_assets, results
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499)
