def test_run_parallel_moment_noref_fextractor(self): print 'test on running Moment noref feature extractor on NorefAssets in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = NorefAsset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = NorefAsset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractors, results = run_executors_in_parallel( MomentNorefFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['Moment_noref_feature_1st_score'], 61.332006624999984) self.assertAlmostEqual(results[0]['Moment_noref_feature_2nd_score'], 4798.659574041666) self.assertAlmostEqual(results[0]['Moment_noref_feature_var_score'], 1036.8371843488285) self.assertAlmostEqual(results[1]['Moment_noref_feature_1st_score'], 59.788567297525134) self.assertAlmostEqual(results[1]['Moment_noref_feature_2nd_score'], 4696.668388042271) self.assertAlmostEqual(results[1]['Moment_noref_feature_var_score'], 1121.519917231207)
def test_run_parallel_moment_fextractor(self): print 'test on running Moment feature extractor in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) self.fextractors, results = run_executors_in_parallel( MomentFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['Moment_feature_ref1st_score'], 59.788567297525134) self.assertAlmostEqual(results[0]['Moment_feature_ref2nd_score'], 4696.668388042269) self.assertAlmostEqual(results[0]['Moment_feature_refvar_score'], 1121.519917231203) self.assertAlmostEqual(results[0]['Moment_feature_dis1st_score'], 61.332006624999984) self.assertAlmostEqual(results[0]['Moment_feature_dis2nd_score'], 4798.659574041666) self.assertAlmostEqual(results[0]['Moment_feature_disvar_score'], 1036.837184348847) self.assertAlmostEqual(results[1]['Moment_feature_ref1st_score'], 59.788567297525134) self.assertAlmostEqual(results[1]['Moment_feature_ref2nd_score'], 4696.668388042269) self.assertAlmostEqual(results[1]['Moment_feature_refvar_score'], 1121.519917231203) self.assertAlmostEqual(results[1]['Moment_feature_dis1st_score'], 59.788567297525134) self.assertAlmostEqual(results[1]['Moment_feature_dis2nd_score'], 4696.668388042269) self.assertAlmostEqual(results[1]['Moment_feature_disvar_score'], 1121.519917231203)
def test_run_parallel_ssim_fextractor(self): print 'test on running SSIM feature extractor in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractors, results = run_executors_in_parallel( SsimFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_score'], 0.857692208333, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_l_score'], 0.9923925625, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_c_score'], 0.9611274375, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_s_score'], 0.897844791667, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_l_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_c_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_s_score'], 1.0, places=4)
def test_run_parallel_psnr_runner(self): print 'test on running PSNR quality runner in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.runners, results = run_executors_in_parallel( PsnrQualityRunner, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None ) self.assertAlmostEqual(results[0]['PSNR_score'], 30.755063979166664) self.assertAlmostEqual(results[1]['PSNR_score'], 60.0)
def test_run_parallel_ssim_fextractor(self): print 'test on running SSIM feature extractor in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractors, results = run_executors_in_parallel( SsimFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_score'], 0.86325137500000004, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_l_score'], 0.99814749999999997, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_c_score'], 0.96132239583333323, places=4) self.assertAlmostEqual(results[0]['SSIM_feature_ssim_s_score'], 0.89770760416666662, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_l_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_c_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['SSIM_feature_ssim_s_score'], 1.0, places=4)
def setUp(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_noisy.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.raw_video_h5py_filepath = config.ROOT + '/workspace/workdir/rawvideo.hdf5' self.raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file( self.raw_video_h5py_filepath) optional_dict2 = {'h5py_file': self.raw_video_h5py_file} _, self.features = run_executors_in_parallel( DisYUVRawVideoExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize= False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor result_store=None, optional_dict=None, optional_dict2=optional_dict2, ) np.random.seed(0) np.random.shuffle(self.features) self.patch_h5py_filepath = config.ROOT + '/workspace/workdir/patch.hdf5' self.model_filename = config.ROOT + "/workspace/model/test_save_load.pkl" NeuralNetTrainTestModel.reset()
def test_run_parallel_psnr_runner(self): print 'test on running PSNR quality runner in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) self.runners, results = run_executors_in_parallel( PsnrQualityRunner, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None) self.assertAlmostEqual(results[0]['PSNR_score'], 30.755063979166664) self.assertAlmostEqual(results[1]['PSNR_score'], 60.0)
def test_run_parallel_ms_ssim_fextractor(self): print 'test on running MS-SSIM feature extractor in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractors, results = run_executors_in_parallel( MsSsimFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.96324620833333319, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9981474999999999, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.9613223958333336, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.8977076041666665, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.9989961250000002, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9858215416666668, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.9411672708333335, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.9992356458333332, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.9970406458333333, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.9779967291666667, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.9992921041666665, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.9995884375000003, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.9938731666666668, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.99940356249999995, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.99990762500000008, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.99822306250000004, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_parallel_ms_ssim_fextractor(self): print 'test on running MS-SSIM feature extractor in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractors, results = run_executors_in_parallel( MsSsimFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_score'], 0.9632498125, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 0.9923925625, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 0.9611274375, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 0.897844791667, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 0.9954706875, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 0.9857694375, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 0.941185875, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 0.997437458333, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 0.997034020833, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 0.977992145833, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 0.998035583333, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 0.999588104167, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 0.99387125, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 0.9995495, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 0.999907625, places=4) self.assertAlmostEqual(results[0]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 0.998222583333, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale0_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale1_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale2_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale3_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_l_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_c_scale4_score'], 1., places=4) self.assertAlmostEqual(results[1]['MS_SSIM_feature_ms_ssim_s_scale4_score'], 1., places=4)
def test_run_parallel_vamf_fextractor(self): print 'test on running VMAF feature extractor in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499, places=4)
def test_run_parallel_dis_y_fextractor(self): print 'test on running dis YUV raw video extractor in parallel (disabled)...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath) optional_dict2 = {'h5py_file': h5py_file} self.fextractors, results = run_executors_in_parallel( DisYUVRawVideoExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=False, # Can't run parallel: can't pickle FileID objects result_store=None, optional_dict={'channels': 'yu'}, optional_dict2=optional_dict2) self.assertAlmostEqual(np.mean(results[0]['dis_y']), 61.332006579182384, places=4) self.assertAlmostEquals(np.mean(results[1]['dis_y']), 59.788567297525148, places=4) self.assertAlmostEqual(np.mean(results[0]['dis_u']), 115.23227407335962, places=4) self.assertAlmostEquals(np.mean(results[1]['dis_u']), 114.49701717535437, places=4) with self.assertRaises(KeyError): np.mean(results[0]['dis_v']) DisYUVRawVideoExtractor.close_h5py_file(h5py_file)
def test_explain_train_test_model(self): model_class = SklearnRandomForestTrainTestModel train_dataset_path = config.ROOT + '/python/test/resource/' \ 'test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) _, self.features = run_executors_in_parallel( MomentNorefFeatureExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, optional_dict=None, optional_dict2=None, ) xys = model_class.get_xys_from_results(self.features[:7]) model = model_class({'norm_type':'normalize', 'random_state':0}, None) model.train(xys) np.random.seed(0) xs = model_class.get_xs_from_results(self.features[7:]) explainer = LocalExplainer(neighbor_samples=1000) exps = explainer.explain(model, xs) self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4) self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4) self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4) self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4) self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4) self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4) self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4) self.assertEqual(exps['feature_names'], ['Moment_noref_feature_1st_score', 'Moment_noref_feature_2nd_score', 'Moment_noref_feature_var_score'] )
def test_run_parallel_vamf_runner_with_model(self): print 'test on running VMAF quality runner in parallel with custom model...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) self.runners, results = run_executors_in_parallel( VmafQualityRunner, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, optional_dict={ 'model_filepath': config.ROOT + "/resource/model/nflx_vmaff_rf_v1.pkl", }) self.assertAlmostEqual(results[0]['VMAF_score'], 73.79861111111113) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[1]['VMAF_score'], 98.22048611111109) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def test_run_parallel_vmaf_legacy_runner(self): print 'test on running VMAF (legacy) quality runner in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) self.runners, results = run_executors_in_parallel( VmafLegacyQualityRunner, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None) self.assertAlmostEqual(results[0]['VMAF_legacy_score'], 60.27316952679754) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[1]['VMAF_legacy_score'], 95.65756240092573) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def setUp(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) _, self.features = run_executors_in_parallel( MomentNorefFeatureExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, optional_dict=None, optional_dict2=None, )
def test_run_parallel_moment_fextractor(self): print "test on running Moment feature extractor in parallel..." ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset( dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={"width": 576, "height": 324}, ) asset_original = Asset( dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={"width": 576, "height": 324}, ) self.fextractors, results = run_executors_in_parallel( MomentFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]["Moment_feature_ref1st_score"], 59.788567297525134, places=4) self.assertAlmostEqual(results[0]["Moment_feature_ref2nd_score"], 4696.668388042269, places=4) self.assertAlmostEqual(results[0]["Moment_feature_refvar_score"], 1121.519917231203, places=4) self.assertAlmostEqual(results[0]["Moment_feature_dis1st_score"], 61.332006624999984, places=4) self.assertAlmostEqual(results[0]["Moment_feature_dis2nd_score"], 4798.659574041666, places=4) self.assertAlmostEqual(results[0]["Moment_feature_disvar_score"], 1036.837184348847, places=4) self.assertAlmostEqual(results[1]["Moment_feature_ref1st_score"], 59.788567297525134, places=4) self.assertAlmostEqual(results[1]["Moment_feature_ref2nd_score"], 4696.668388042269, places=4) self.assertAlmostEqual(results[1]["Moment_feature_refvar_score"], 1121.519917231203, places=4) self.assertAlmostEqual(results[1]["Moment_feature_dis1st_score"], 59.788567297525134, places=4) self.assertAlmostEqual(results[1]["Moment_feature_dis2nd_score"], 4696.668388042269, places=4) self.assertAlmostEqual(results[1]["Moment_feature_disvar_score"], 1121.519917231203, places=4)
def run(self): """ Do all the calculation here. :return: """ # for each FeatureExtractor_type key in feature_dict, find the subclass # of FeatureExtractor, run, and put results in a dict for fextractor_type in self.feature_dict: # fextractor = self._get_fextractor_instance(fextractor_type) # fextractor.run() # results = fextractor.results fextractor_class = FeatureExtractor.find_subclass(fextractor_type) _, results = run_executors_in_parallel( fextractor_class, assets=self.assets, fifo_mode=self.fifo_mode, delete_workdir=self.delete_workdir, parallelize=self.parallelize, result_store=self.result_store, optional_dict=self.optional_dict, optional_dict2=self.optional_dict2, ) self.type2results_dict[fextractor_type] = results # assemble an output dict with demanded atom features # atom_features_dict = self.fextractor_atom_features_dict result_dicts = [dict() for _ in self.assets] for fextractor_type in self.feature_dict: assert fextractor_type in self.type2results_dict for atom_feature in self._get_atom_features(fextractor_type): scores_key = self._get_scores_key(fextractor_type, atom_feature) for result_index, result in enumerate( self.type2results_dict[fextractor_type]): result_dicts[result_index][scores_key] = result[scores_key] self.results = map( lambda (asset, result_dict): BasicResult(asset, result_dict), zip(self.assets, result_dicts))
def setUp(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.h5py_filepath = config.ROOT + '/workspace/workdir/test.hdf5' self.h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath) optional_dict2 = {'h5py_file': self.h5py_file} _, self.features = run_executors_in_parallel( DisYUVRawVideoExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor result_store=None, optional_dict=None, optional_dict2=optional_dict2, )
def run(self): """ Do all the calculation here. :return: """ # for each FeatureExtractor_type key in feature_dict, find the subclass # of FeatureExtractor, run, and put results in a dict for fextractor_type in self.feature_dict: # fextractor = self._get_fextractor_instance(fextractor_type) # fextractor.run() # results = fextractor.results fextractor_class = FeatureExtractor.find_subclass(fextractor_type) _, results = run_executors_in_parallel( fextractor_class, assets=self.assets, fifo_mode=self.fifo_mode, delete_workdir=self.delete_workdir, parallelize=self.parallelize, result_store=self.result_store, optional_dict=self.optional_dict, optional_dict2=self.optional_dict2, ) self.type2results_dict[fextractor_type] = results # assemble an output dict with demanded atom features # atom_features_dict = self.fextractor_atom_features_dict result_dicts = [dict() for _ in self.assets] for fextractor_type in self.feature_dict: assert fextractor_type in self.type2results_dict for atom_feature in self._get_atom_features(fextractor_type): scores_key = self._get_scores_key(fextractor_type, atom_feature) for result_index, result in enumerate(self.type2results_dict[ fextractor_type]): result_dicts[result_index][scores_key] = result[scores_key] self.results = map( lambda (asset, result_dict): BasicResult(asset, result_dict), zip(self.assets, result_dicts) )
def test_run_parallel_ssim_fextractor(self): print "test on running SSIM feature extractor in parallel..." ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset( dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={"width": 576, "height": 324}, ) asset_original = Asset( dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={"width": 576, "height": 324}, ) self.fextractors, results = run_executors_in_parallel( SsimFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]["SSIM_feature_ssim_score"], 0.86322654166666657, places=4) self.assertAlmostEqual(results[0]["SSIM_feature_ssim_l_score"], 0.9981474583333334, places=4) self.assertAlmostEqual(results[0]["SSIM_feature_ssim_c_score"], 0.96126793750000006, places=4) self.assertAlmostEqual(results[0]["SSIM_feature_ssim_s_score"], 0.89773633333333336, places=4) self.assertAlmostEqual(results[1]["SSIM_feature_ssim_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["SSIM_feature_ssim_l_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["SSIM_feature_ssim_c_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["SSIM_feature_ssim_s_score"], 1.0, places=4)
def test_run_parallel_dis_y_fextractor(self): print 'test on running dis YUV raw video extractor in parallel (disabled)...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=2, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath) optional_dict2 = {'h5py_file': h5py_file} self.fextractors, results = run_executors_in_parallel( DisYUVRawVideoExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=False, # Can't run parallel: can't pickle FileID objects result_store=None, optional_dict={'channels': 'yu'}, optional_dict2=optional_dict2 ) self.assertAlmostEqual(np.mean(results[0]['dis_y']), 61.332006579182384, places=4) self.assertAlmostEquals(np.mean(results[1]['dis_y']), 59.788567297525148, places=4) self.assertAlmostEqual(np.mean(results[0]['dis_u']), 115.23227407335962, places=4) self.assertAlmostEquals(np.mean(results[1]['dis_u']), 114.49701717535437, places=4) with self.assertRaises(KeyError): np.mean(results[0]['dis_v']) DisYUVRawVideoExtractor.close_h5py_file(h5py_file)
def test_run_parallel_vamf_runner_with_model(self): print 'test on running VMAF quality runner in parallel with custom model...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.runners, results = run_executors_in_parallel( VmafQualityRunner, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, optional_dict={ 'model_filepath':config.ROOT + "/resource/model/nflx_vmaff_rf_v1.pkl", } ) self.assertAlmostEqual(results[0]['VMAF_score'], 73.79861111111113) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[1]['VMAF_score'], 98.22048611111109) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def test_run_parallel_vmaf_legacy_runner(self): print 'test on running VMAF (legacy) quality runner in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) self.runners, results = run_executors_in_parallel( VmafLegacyQualityRunner, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None ) self.assertAlmostEqual(results[0]['VMAF_legacy_score'], 60.27316952679754) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[1]['VMAF_legacy_score'], 95.65756240092573) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def main(): if len(sys.argv) < 2: print_usage() return 2 input_filepath = sys.argv[1] model_path = get_cmd_option(sys.argv, 2, len(sys.argv), '--model') out_fmt = get_cmd_option(sys.argv, 2, len(sys.argv), '--out-fmt') if not (out_fmt is None or out_fmt == 'xml' or out_fmt == 'json' or out_fmt == 'text'): print_usage() return 2 pool_method = get_cmd_option(sys.argv, 2, len(sys.argv), '--pool') if not (pool_method is None or pool_method in POOL_METHODS): print_usage() return 2 parallelize = cmd_option_exists(sys.argv, 2, len(sys.argv), '--parallelize') assets = [] line_idx = 0 with open(input_filepath, "rt") as input_file: for line in input_file.readlines(): # match comment mo = re.match(r"^#", line) if mo: print "Skip commented line: {}".format(line) continue # match whitespace mo = re.match(r"[\s]+", line) if mo: continue # example: yuv420p 576 324 ref.yuv dis.yuv mo = re.match(r"([\S]+) ([0-9]+) ([0-9]+) ([\S]+) ([\S]+)", line) if not mo or mo.group(1) not in FMTS: print "Unknown format: {}".format(line) print_usage() return 1 fmt = mo.group(1) width = int(mo.group(2)) height = int(mo.group(3)) ref_file = mo.group(4) dis_file = mo.group(5) asset = Asset(dataset="cmd", content_id=0, asset_id=line_idx, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_file, dis_path=dis_file, asset_dict={ 'width': width, 'height': height, 'yuv_type': fmt }) assets.append(asset) line_idx += 1 runner_class = VmafQualityRunner if model_path is None: optional_dict = None else: optional_dict = {'model_filepath': model_path} # construct an VmafQualityRunner object to assert assets, and to remove _ = runner_class( assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=optional_dict, optional_dict2=None, ) runners, results = run_executors_in_parallel( runner_class, assets, fifo_mode=True, delete_workdir=True, parallelize=parallelize, result_store=None, optional_dict=optional_dict, optional_dict2=None, ) # output for result in results: # pooling if pool_method == 'harmonic_mean': result.set_score_aggregate_method(ListStats.harmonic_mean) if pool_method == 'min': result.set_score_aggregate_method(np.min) else: # None or 'mean' pass if out_fmt == 'xml': print result.to_xml() elif out_fmt == 'json': print result.to_json() else: # None or 'json' print '============================' print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id) print '============================' print str(result) return 0
# shuffle assets np.random.seed(seed) np.random.shuffle(assets) assets = assets[:(num_train + num_test)] raw_video_h5py_filepath = config.ROOT + '/workspace/workdir/rawvideo.hdf5' raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file(raw_video_h5py_filepath) print '======================== Extract raw YUVs ==============================' _, raw_yuvs = run_executors_in_parallel( DisYUVRawVideoExtractor, assets, fifo_mode=True, delete_workdir=True, parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor result_store=None, optional_dict=None, optional_dict2={'h5py_file': raw_video_h5py_file}) patch_h5py_filepath = config.ROOT + '/workspace/workdir/patch.hdf5' patch_h5py_file = ToddNoiseClassifierTrainTestModel.open_h5py_file(patch_h5py_filepath) model = ToddNoiseClassifierTrainTestModel( param_dict={ 'seed': seed, 'n_epochs': n_epochs, }, logger=None, optional_dict2={ # for options that won't impact the result # 'checkpoints_dir': config.ROOT + '/workspace/checkpoints_dir',
def test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean): test_assets = read_dataset(test_dataset) optional_dict = {'model_filepath': model_filepath} # construct an quality runner object to assert assets only runner = runner_class( test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, ) try: # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, ) for result in results: result.set_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) stats = TrainTestModel.get_stats(groundtruths, predictions) print 'Stats on testing data: {}'.format( TrainTestModel.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) TrainTestModel.plot_scatter(ax, stats, content_ids) ax.set_xlabel('DMOS') ax.set_ylabel("Predicted Score") ax.grid() # ax.set_title( "Dataset: {dataset}, Runner: {runner}\n{stats}".format( ax.set_title("{runner}\n{stats}".format( dataset=test_assets[0].dataset, # runner=results[0].executor_id, runner=runner_class.TYPE, stats=TrainTestModel.format_stats(stats), # stats="", # stats=TrainTestModel.format_stats3(stats), )) except Exception as e: print "Error: " + str(e) return test_assets, results
def main(): if len(sys.argv) < 2: print_usage() return 2 input_filepath = sys.argv[1] model_path = get_cmd_option(sys.argv, 2, len(sys.argv), '--model') out_fmt = get_cmd_option(sys.argv, 2, len(sys.argv), '--out-fmt') if not (out_fmt is None or out_fmt == 'xml' or out_fmt == 'json' or out_fmt == 'text'): print_usage() return 2 pool_method = get_cmd_option(sys.argv, 2, len(sys.argv), '--pool') if not (pool_method is None or pool_method in POOL_METHODS): print '--pool can only have option among {}'.format(', '.join(POOL_METHODS)) return 2 parallelize = cmd_option_exists(sys.argv, 2, len(sys.argv), '--parallelize') assets = [] line_idx = 0 with open(input_filepath, "rt") as input_file: for line in input_file.readlines(): # match comment mo = re.match(r"^#", line) if mo: print "Skip commented line: {}".format(line) continue # match whitespace mo = re.match(r"[\s]+", line) if mo: continue # example: yuv420p 576 324 ref.yuv dis.yuv mo = re.match(r"([\S]+) ([0-9]+) ([0-9]+) ([\S]+) ([\S]+)", line) if not mo or mo.group(1) not in FMTS: print "Unknown format: {}".format(line) print_usage() return 1 fmt = mo.group(1) width = int(mo.group(2)) height = int(mo.group(3)) ref_file = mo.group(4) dis_file = mo.group(5) asset = Asset(dataset="cmd", content_id=0, asset_id=line_idx, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_file, dis_path=dis_file, asset_dict={'width':width, 'height':height, 'yuv_type':fmt} ) assets.append(asset) line_idx += 1 runner_class = VmafQualityRunner if model_path is None: optional_dict = None else: optional_dict = {'model_filepath':model_path} # construct an VmafQualityRunner object to assert assets, and to remove _ = runner_class(assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=optional_dict, optional_dict2=None, ) runners, results = run_executors_in_parallel( runner_class, assets, fifo_mode=True, delete_workdir=True, parallelize=parallelize, result_store=None, optional_dict=optional_dict, optional_dict2=None, ) # output for result in results: # pooling if pool_method == 'harmonic_mean': result.set_score_aggregate_method(ListStats.harmonic_mean) elif pool_method == 'min': result.set_score_aggregate_method(np.min) elif pool_method == 'median': result.set_score_aggregate_method(np.median) elif pool_method == 'perc5': result.set_score_aggregate_method(ListStats.perc5) elif pool_method == 'perc10': result.set_score_aggregate_method(ListStats.perc10) elif pool_method == 'perc20': result.set_score_aggregate_method(ListStats.perc20) else: # None or 'mean' pass if out_fmt == 'xml': print result.to_xml() elif out_fmt == 'json': print result.to_json() else: # None or 'json' print '============================' print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id) print '============================' print str(result) return 0
runner = runner_class( assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=optional_dict, ) try: # run runners, results = run_executors_in_parallel( runner_class, assets, fifo_mode=True, delete_workdir=True, parallelize=parallelize, result_store=None, optional_dict=optional_dict, ) # output for result in results: print '============================' print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id) print '============================' print str(result) finally: pass
def test_on_dataset( test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean, type="regressor", **kwargs ): if type == "regressor": model_type = RegressorMixin elif type == "classifier": model_type = ClassifierMixin else: assert False test_assets = read_dataset(test_dataset, **kwargs) test_raw_assets = None try: for test_asset in test_assets: assert test_asset.groundtruth is not None except AssertionError: # no groundtruth, try do subjective modeling subj_model_class = ( kwargs["subj_model_class"] if "subj_model_class" in kwargs and kwargs["subj_model_class"] is not None else DmosModel ) subjective_model = subj_model_class(RawDatasetReader(test_dataset)) subjective_model.run_modeling(**kwargs) test_dataset_aggregate = subjective_model.to_aggregated_dataset(**kwargs) test_raw_assets = test_assets test_assets = read_dataset(test_dataset_aggregate, **kwargs) if model_filepath is not None: optional_dict = {"model_filepath": model_filepath} else: optional_dict = None # construct an quality runner object to assert assets only runner = runner_class( test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) for result in results: result.set_score_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) raw_grountruths = None if test_raw_assets is None else map(lambda asset: asset.raw_groundtruth, test_raw_assets) stats = model_type.get_stats(groundtruths, predictions, ys_label_raw=raw_grountruths) print "Stats on testing data: {}".format(model_type.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) model_type.plot_scatter(ax, stats, content_ids) ax.set_xlabel("True Score") ax.set_ylabel("Predicted Score") ax.grid() ax.set_title( "{runner}\n{stats}".format( dataset=test_assets[0].dataset, runner=runner_class.TYPE, stats=model_type.format_stats(stats) ) ) return test_assets, results
def test_run_parallel_vamf_fextractor_with_result_store(self): print 'test on running VMAF feature extractor with result store ' \ 'in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={ 'width': 576, 'height': 324 }) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={ 'width': 576, 'height': 324 }) result_store = FileSystemResultStore(logger=None) print ' running for the first time with fresh calculation...' self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue( os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue( os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499)
def test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean, type='regressor', **kwargs): if type == 'regressor': model_type = RegressorMixin elif type == 'classifier': model_type = ClassifierMixin else: assert False test_assets = read_dataset(test_dataset, **kwargs) if model_filepath is not None: optional_dict = {'model_filepath': model_filepath} else: optional_dict = None # construct an quality runner object to assert assets only runner = runner_class( test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) try: # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) for result in results: result.set_score_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) stats = model_type.get_stats(groundtruths, predictions) print 'Stats on testing data: {}'.format( model_type.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) model_type.plot_scatter(ax, stats, content_ids) ax.set_xlabel('True Score') ax.set_ylabel("Predicted Score") ax.grid() ax.set_title("{runner}\n{stats}".format( dataset=test_assets[0].dataset, runner=runner_class.TYPE, stats=model_type.format_stats(stats), )) except Exception as e: print "Error: " + str(e) raise e return test_assets, results
def test_run_parallel_vamf_fextractor_with_result_store(self): print "test on running VMAF feature extractor with result store " "in parallel..." ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset( dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={"width": 576, "height": 324}, ) asset_original = Asset( dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={"width": 576, "height": 324}, ) result_store = FileSystemResultStore(logger=None) print " running for the first time with fresh calculation..." self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print " running for the second time with stored results..." _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]["VMAF_feature_vif_score"], 0.4460930625, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_motion_score"], 4.04982535417, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm2_score"], 0.925421075027, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_ansnr_score"], 23.5095715208, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_num_score"], 712650.023478, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_den_score"], 1597314.95249, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_num_score"], 6899.24648475, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_den_score"], 7535.29963308, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_anpsnr_score"], 34.164776875, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale0_score"], 0.363420489439, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale1_score"], 0.766647542135, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale2_score"], 0.862854666902, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale3_score"], 0.915971778036, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale0_score"], 0.86711114907261122, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale1_score"], 0.87407085185224942, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale2_score"], 0.92448300466700284, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale3_score"], 0.96367203202196239, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_vif2_score"], 0.72722361912801026, places=4) self.assertAlmostEqual(results[0]["VMAF_feature_adm3_score"], 0.90733425940345658, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_motion_score"], 4.04982535417, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_ansnr_score"], 31.2714392708, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_num_score"], 1597314.86733, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_den_score"], 1597314.95249, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_num_score"], 7535.29963308, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_den_score"], 7535.29963308, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_anpsnr_score"], 41.9266444375, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_vif2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["VMAF_feature_adm3_score"], 1.0, places=4)
def test_run_parallel_ms_ssim_fextractor(self): print "test on running MS-SSIM feature extractor in parallel..." ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset( dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={"width": 576, "height": 324}, ) asset_original = Asset( dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={"width": 576, "height": 324}, ) self.fextractors, results = run_executors_in_parallel( MsSsimFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, ) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_score"], 0.9632498125, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale0_score"], 0.9981474583333334, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale0_score"], 0.96126793750000006, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale0_score"], 0.89773633333333336, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale1_score"], 0.99899612500000001, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale1_score"], 0.9857694375, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale1_score"], 0.941185875, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale2_score"], 0.99923564583333324, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale2_score"], 0.997034020833, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale2_score"], 0.977992145833, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale3_score"], 0.99929210416666658, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale3_score"], 0.999588104167, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale3_score"], 0.99387125, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_l_scale4_score"], 0.99940356249999995, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_c_scale4_score"], 0.999907625, places=4) self.assertAlmostEqual(results[0]["MS_SSIM_feature_ms_ssim_s_scale4_score"], 0.998222583333, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale0_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale1_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale2_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale3_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_l_scale4_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_c_scale4_score"], 1.0, places=4) self.assertAlmostEqual(results[1]["MS_SSIM_feature_ms_ssim_s_scale4_score"], 1.0, places=4)
def test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean): test_assets = read_dataset(test_dataset) optional_dict = { 'model_filepath':model_filepath } # construct an quality runner object to assert assets only runner = runner_class(test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, ) try: # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, ) for result in results: result.set_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) stats = TrainTestModel.get_stats(groundtruths, predictions) print 'Stats on testing data: {}'.format(TrainTestModel.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) TrainTestModel.plot_scatter(ax, stats, content_ids) ax.set_xlabel('DMOS') ax.set_ylabel("Predicted Score") ax.grid() # ax.set_title( "Dataset: {dataset}, Runner: {runner}\n{stats}".format( ax.set_title( "{runner}\n{stats}".format( dataset=test_assets[0].dataset, # runner=results[0].executor_id, runner=runner_class.TYPE, stats=TrainTestModel.format_stats(stats), # stats="", # stats=TrainTestModel.format_stats3(stats), )) except Exception as e: print "Error: " + str(e) return test_assets, results
# construct an VmafQualityRunner object to assert assets, and to remove runner = runner_class(assets, None, fifo_mode=True, delete_workdir=True, result_store=None, optional_dict=optional_dict, ) try: # run runners, results = run_executors_in_parallel( runner_class, assets, fifo_mode=True, delete_workdir=True, parallelize=parallelize, result_store=None, optional_dict=optional_dict, ) # output for result in results: print '============================' print 'Asset {asset_id}:'.format(asset_id=result.asset.asset_id) print '============================' print str(result) finally: pass
def test_run_parallel_vamf_fextractor_with_result_store(self): print 'test on running VMAF feature extractor with result store ' \ 'in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) result_store = FileSystemResultStore(logger=None) print ' running for the first time with fresh calculation...' self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.925421075027, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.24648475, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.29963308, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.86711114907261122, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.87407085185224942, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.92448300466700284, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96367203202196239, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4) self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.90733425940345658, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.29963308, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.29963308, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4) self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
def test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean, type='regressor', **kwargs): if type == 'regressor': model_type = RegressorMixin elif type == 'classifier': model_type = ClassifierMixin else: assert False test_assets = read_dataset(test_dataset, **kwargs) test_raw_assets = None try: for test_asset in test_assets: assert test_asset.groundtruth is not None except AssertionError: # no groundtruth, try do subjective modeling subj_model_class = kwargs[ 'subj_model_class'] if 'subj_model_class' in kwargs and kwargs[ 'subj_model_class'] is not None else DmosModel subjective_model = subj_model_class(RawDatasetReader(test_dataset)) subjective_model.run_modeling(**kwargs) test_dataset_aggregate = subjective_model.to_aggregated_dataset( **kwargs) test_raw_assets = test_assets test_assets = read_dataset(test_dataset_aggregate, **kwargs) if model_filepath is not None: optional_dict = {'model_filepath': model_filepath} else: optional_dict = None # construct an quality runner object to assert assets only runner = runner_class( test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) for result in results: result.set_score_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) raw_grountruths = None if test_raw_assets is None else \ map(lambda asset: asset.raw_groundtruth, test_raw_assets) stats = model_type.get_stats(groundtruths, predictions, ys_label_raw=raw_grountruths) print 'Stats on testing data: {}'.format(model_type.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) model_type.plot_scatter(ax, stats, content_ids) ax.set_xlabel('True Score') ax.set_ylabel("Predicted Score") ax.grid() ax.set_title("{runner}\n{stats}".format( dataset=test_assets[0].dataset, runner=runner_class.TYPE, stats=model_type.format_stats(stats), )) return test_assets, results
def test_run_parallel_vamf_fextractor_with_result_store(self): print 'test on running VMAF feature extractor with result store ' \ 'in parallel...' ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv" dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv" asset = Asset(dataset="test", content_id=0, asset_id=0, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=dis_path, asset_dict={'width':576, 'height':324}) asset_original = Asset(dataset="test", content_id=0, asset_id=1, workdir_root=config.ROOT + "/workspace/workdir", ref_path=ref_path, dis_path=ref_path, asset_dict={'width':576, 'height':324}) result_store = FileSystemResultStore(logger=None) print ' running for the first time with fresh calculation...' self.fextractors, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) result0, result1 = results self.assertTrue(os.path.exists(result_store._get_result_file_path(result0))) self.assertTrue(os.path.exists(result_store._get_result_file_path(result1))) print ' running for the second time with stored results...' _, results = run_executors_in_parallel( VmafFeatureExtractor, [asset, asset_original], fifo_mode=True, delete_workdir=True, parallelize=True, result_store=result_store, ) self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313) self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141) self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329) self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038) self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996) self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836) self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499) self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089) self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212) self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667) self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0) self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322) self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417) self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996) self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499) self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499) self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067) self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499)