class ResultStoreTest(unittest.TestCase):
def setUp(self):
ref_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_0_0.yuv"
dis_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_1_0.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':1920, 'height':1080})
self.runner = VmafLegacyQualityRunner(
[asset], None, fifo_mode=True,
delete_workdir=True, result_store=None,
)
self.runner.run()
self.result = self.runner.results[0]
def tearDown(self):
if hasattr(self, 'result') and hasattr(self, 'result_store'):
self.result_store.delete(self.result.asset, self.result.executor_id)
pass
def test_file_system_result_store_save_load(self):
print 'test on file system result store save and load...'
self.result_store = FileSystemResultStore(logger=None)
asset = self.result.asset
executor_id = self.result.executor_id
self.result_store.save(self.result)
loaded_result = self.result_store.load(asset, executor_id)
self.assertEquals(self.result, loaded_result)
class ResultStoreTest(unittest.TestCase):
def setUp(self):
ref_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_0_0.yuv"
dis_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_1_0.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':1920, 'height':1080})
self.runner = VmafLegacyQualityRunner(
[asset], None, fifo_mode=True,
delete_workdir=True, result_store=None,
)
self.runner.run()
self.result = self.runner.results[0]
def tearDown(self):
if hasattr(self, 'result') and hasattr(self, 'result_store'):
self.result_store.delete(self.result.asset, self.result.executor_id)
pass
def test_file_system_result_store_save_load(self):
print 'test on file system result store save and load...'
self.result_store = FileSystemResultStore(logger=None)
asset = self.result.asset
executor_id = self.result.executor_id
self.result_store.save(self.result)
loaded_result = self.result_store.load(asset, executor_id)
self.assertEquals(self.result, loaded_result)
def test_file_system_result_store_save_load(self):
print 'test on file system result store save and load...'
self.result_store = FileSystemResultStore(logger=None)
asset = self.result.asset
executor_id = self.result.executor_id
self.result_store.save(self.result)
loaded_result = self.result_store.load(asset, executor_id)
self.assertEquals(self.result, loaded_result)
def test_run_vamf_legacy_runner_with_result_store(self):
print 'test on running VMAF (legacy) runner with result store...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
self.runner = VmafLegacyQualityRunner(
[asset, asset_original],
None, fifo_mode=True,
delete_workdir=True,
result_store=result_store
)
print ' running for the first time with fresh calculation...'
self.runner.run()
result0, result1 = self.runner.results
# NOTE: since stored results are actually VMAF_feature's not VMAF's,
# the two paths below shouldn't exist
self.assertFalse(os.path.exists(result_store._get_result_file_path(result0)))
self.assertFalse(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAF_legacy_score'], 60.27316952679754)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'], 0.9155242291666666)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_legacy_score'], 95.65756240092573)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
try:
dataset = import_python_file(dataset_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
try:
runner_class = QualityRunner.find_subclass(quality_type)
except:
print_usage()
return 2
result_store = FileSystemResultStore()
run_remove_results_for_dataset(result_store, dataset, runner_class)
return 0
def run_vmaf_cv(train_dataset_filepath,
test_dataset_filepath,
param_filepath,
output_model_filepath=None,
**kwargs):
logger = get_stdout_logger()
result_store = FileSystemResultStore()
train_dataset = import_python_file(train_dataset_filepath)
test_dataset = import_python_file(
test_dataset_filepath) if test_dataset_filepath is not None else None
param = import_python_file(param_filepath)
# === plot scatter ===
nrows = 1
ncols = 2
fig, axs = plt.subplots(figsize=(5 * ncols, 5 * nrows),
nrows=nrows,
ncols=ncols)
train_test_vmaf_on_dataset(train_dataset,
test_dataset,
param,
param,
axs[0],
axs[1],
result_store,
parallelize=True,
logger=None,
output_model_filepath=output_model_filepath,
**kwargs)
if 'xlim' in kwargs:
axs[0].set_xlim(kwargs['xlim'])
axs[1].set_xlim(kwargs['xlim'])
if 'ylim' in kwargs:
axs[0].set_ylim(kwargs['ylim'])
axs[1].set_ylim(kwargs['ylim'])
bbox = {'facecolor': 'white', 'alpha': 1, 'pad': 20}
axs[0].annotate('Training Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
axs[1].annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
plt.tight_layout()
# === clean up ===
close_logger(logger)
def test_file_system_result_store_save_load(self):
print 'test on file system result store save and load...'
self.result_store = FileSystemResultStore(logger=None)
asset = self.result.asset
executor_id = self.result.executor_id
self.result_store.save(self.result)
loaded_result = self.result_store.load(asset, executor_id)
self.assertEquals(self.result, loaded_result)
def setUp(self):
ref_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_0_0.yuv"
dis_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_1_0.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':1920, 'height':1080})
self.runner = VmafLegacyQualityRunner(
[asset], None, fifo_mode=True,
delete_workdir=True, result_store=FileSystemResultStore(),
)
self.runner.run()
self.result = self.runner.results[0]
def explain_model_on_dataset(model, test_assets_selected_indexs,
test_dataset_filepath):
def print_assets(test_assets):
print '\n'.join(
map(
lambda (i, asset): "Asset {i}: {name}".format(
i=i, name=get_file_name_without_extension(asset.dis_path)),
enumerate(test_assets)))
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print "Assets selected for local explanation: {}".format(
test_assets_selected_indexs)
result_store = FileSystemResultStore()
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict['feature_dict'],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
plt.show()
def run_vmaf_kfold_cv(dataset_filepath,
contentid_groups,
param_filepath,
aggregate_method,
):
logger = get_stdout_logger()
result_store = FileSystemResultStore()
dataset = import_python_file(dataset_filepath)
param = import_python_file(param_filepath)
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
cv_on_dataset(dataset, param, param, ax, result_store, contentid_groups,
logger, aggregate_method)
ax.set_xlim([0, 120])
ax.set_ylim([0, 120])
plt.tight_layout()
# === clean up ===
close_logger(logger)
def test_run_parallel_vamf_fextractor_with_result_store(self):
print "test on running VMAF feature extractor with result store " "in parallel..."
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(
dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={"width": 576, "height": 324},
)
asset_original = Asset(
dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={"width": 576, "height": 324},
)
result_store = FileSystemResultStore(logger=None)
print " running for the first time with fresh calculation..."
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print " running for the second time with stored results..."
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_score"], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_motion_score"], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm2_score"], 0.925421075027, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_ansnr_score"], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_num_score"], 712650.023478, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_den_score"], 1597314.95249, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_num_score"], 6899.24648475, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_den_score"], 7535.29963308, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_anpsnr_score"], 34.164776875, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale0_score"], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale1_score"], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale2_score"], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif_scale3_score"], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale0_score"], 0.86711114907261122, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale1_score"], 0.87407085185224942, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale2_score"], 0.92448300466700284, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm_scale3_score"], 0.96367203202196239, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_vif2_score"], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]["VMAF_feature_adm3_score"], 0.90733425940345658, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_motion_score"], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_ansnr_score"], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_num_score"], 1597314.86733, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_den_score"], 1597314.95249, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_num_score"], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_den_score"], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_anpsnr_score"], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale0_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale1_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm_scale3_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_vif2_score"], 1.0, places=4)
self.assertAlmostEqual(results[1]["VMAF_feature_adm3_score"], 1.0, places=4)
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.925421075027, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.24648475, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.29963308, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.86711114907261122, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.87407085185224942, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.92448300466700284, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96367203202196239, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.90733425940345658, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.29963308, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
def setUp(self):
self.result_store = FileSystemResultStore()
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 644527.3311971038, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 6899.815530270836, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.15266368750002, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.3655846219305399, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.7722301581694561, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.8681486658208089, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.9207121810522212, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1449635.3522745417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1449635.3812459996, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 7535.801140312499, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.65012097916668, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0000000132944864, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 0.9999998271651448, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 0.9999998649680067, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 0.9999998102499, places=4)
def test_run_vamf_legacy_runner_with_result_store(self):
print 'test on running VMAF (legacy) runner with result store...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
result_store = FileSystemResultStore(logger=None)
self.runner = VmafLegacyQualityRunner([asset, asset_original],
None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store)
print ' running for the first time with fresh calculation...'
self.runner.run()
result0, result1 = self.runner.results
# NOTE: since stored results are actually VMAF_feature's not VMAF's,
# the two paths below shouldn't exist
self.assertFalse(
os.path.exists(result_store._get_result_file_path(result0)))
self.assertFalse(
os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.runner.run()
results = self.runner.results
self.assertAlmostEqual(results[0]['VMAF_legacy_score'],
60.27316952679754,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_score'],
0.9155242291666666,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
22.533456770833329,
places=4)
self.assertAlmostEqual(results[1]['VMAF_legacy_score'],
95.65756240092573,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.030914145833322,
places=4)
feature_param = import_python_file(feature_param_filepath)
model_param = import_python_file(model_param_filepath)
except Exception as e:
print "Error: " + str(e)
exit(1)
if do_parallelize == 'yes':
parallelize = True
elif do_parallelize == 'no':
parallelize = False
else:
print_usage()
exit(2)
if cache_result == 'yes':
result_store = FileSystemResultStore()
elif cache_result == 'no':
result_store = None
else:
print_usage()
exit(2)
try:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
train_test_on_dataset(train_dataset=train_dataset, test_dataset=None,
feature_param=feature_param, model_param=model_param,
train_ax=ax, test_ax=None,
result_store=result_store,
parallelize=parallelize,
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
test_dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv), '--vmaf-model')
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv), '--parallelize')
print_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--print-result')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv), '--suppress-plot')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None
or pool_method in POOL_METHODS):
print '--pool can only have option among {}'.format(', '.join(POOL_METHODS))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print "Error: " + str(e)
return 1
if vmaf_model_path is not None and quality_type != VmafQualityRunner.TYPE:
print "Input error: only quality_type of VMAF accepts --vmaf-model."
print_usage()
return 2
try:
test_dataset = import_python_file(test_dataset_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
try:
runner_class = QualityRunner.find_subclass(quality_type)
except Exception as e:
print "Error: " + str(e)
return 1
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
try:
if suppress_plot:
raise AssertionError
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
assets, results = test_on_dataset(test_dataset, runner_class, ax,
result_store, vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
bbox = {'facecolor':'white', 'alpha':0.5, 'pad':20}
ax.annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
plt.show()
except ImportError:
print_matplotlib_warning()
assets, results = test_on_dataset(test_dataset, runner_class, None,
result_store, vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
except AssertionError:
assets, results = test_on_dataset(test_dataset, runner_class, None,
result_store, vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
if print_result:
for result in results:
print result
print ''
return 0
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractors, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
result0, result1 = results
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
_, results = run_executors_in_parallel(
VmafFeatureExtractor,
[asset, asset_original],
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=result_store,
)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9254334398006141)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
22.533456770833329)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'],
644527.3311971038)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'],
1449635.3812459996)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'],
6899.815530270836)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'],
7535.801140312499)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'],
34.15266368750002)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'],
0.3655846219305399)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'],
0.7722301581694561)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'],
0.8681486658208089)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'],
0.9207121810522212)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.030914145833322)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'],
1449635.3522745417)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'],
1449635.3812459996)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'],
7535.801140312499)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'],
7535.801140312499)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'],
41.65012097916668)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'],
1.0000000132944864)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'],
0.9999998271651448)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'],
0.9999998649680067)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'],
0.9999998102499)
def main():
if len(sys.argv) < 5:
print_usage()
return 2
try:
train_dataset_filepath = sys.argv[1]
feature_param_filepath = sys.argv[2]
model_param_filepath = sys.argv[3]
output_model_filepath = sys.argv[4]
except ValueError:
print_usage()
return 2
try:
train_dataset = import_python_file(train_dataset_filepath)
feature_param = import_python_file(feature_param_filepath)
model_param = import_python_file(model_param_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv), '--parallelize')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None
or pool_method in POOL_METHODS):
print '--pool can only have option among {}'.format(', '.join(POOL_METHODS))
return 2
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
logger = None
try:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
train_test_vmaf_on_dataset(train_dataset=train_dataset, test_dataset=None,
feature_param=feature_param, model_param=model_param,
train_ax=ax, test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method
)
bbox = {'facecolor':'white', 'alpha':0.5, 'pad':20}
ax.annotate('Training Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
plt.show()
except ImportError:
print_matplotlib_warning()
train_test_vmaf_on_dataset(train_dataset=train_dataset, test_dataset=None,
feature_param=feature_param, model_param=model_param,
train_ax=None, test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method
)
return 0
