class ResultStoreTest(unittest.TestCase):
def setUp(self):
ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':1920, 'height':1080})
self.runner = VmafLegacyQualityRunner(
[asset], None, fifo_mode=True,
delete_workdir=True, result_store=None,
)
self.runner.run()
self.result = self.runner.results[0]
def tearDown(self):
if hasattr(self, 'result') and hasattr(self, 'result_store'):
self.result_store.delete(self.result.asset, self.result.executor_id)
pass
def test_file_system_result_store_save_load(self):
print 'test on file system result store save and load...'
self.result_store = FileSystemResultStore(logger=None)
asset = self.result.asset
executor_id = self.result.executor_id
self.result_store.save(self.result)
loaded_result = self.result_store.load(asset, executor_id)
self.assertEquals(self.result, loaded_result)
def test_run_strred_fextractor_blackframes(self):
print 'test on running STRRED feature extractor on flat frames...'
ref_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_10.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
from vmaf.core.result_store import FileSystemResultStore
result_store = FileSystemResultStore(logger=None)
self.fextractor = StrredFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=result_store
)
print ' running for the first time with fresh calculation...'
self.fextractor.run(parallelize=True)
result0, result1 = self.fextractor.results
import os
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.fextractor.run(parallelize=True)
results = self.fextractor.results
# ignore NaN
for result in results:
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'], 1220.5679849999999, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'], 50983.3097155, places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'], 62228595.6081, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'], 0.0, places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'], 0.0, places=4)
def test_file_system_result_store_save_load(self):
print 'test on file system result store save and load...'
self.result_store = FileSystemResultStore(logger=None)
asset = self.result.asset
executor_id = self.result.executor_id
self.result_store.save(self.result)
loaded_result = self.result_store.load(asset, executor_id)
self.assertEquals(self.result, loaded_result)
def explain_model_on_dataset(
model,
test_assets_selected_indexs,
test_dataset_filepath,
result_store_dir=VmafConfig.file_result_store_path()):
def print_assets(test_assets):
print('\n'.join(
map(
lambda tasset: "Asset {i}: {name}".format(
i=tasset[0],
name=get_file_name_without_extension(tasset[1].dis_path)),
enumerate(test_assets))))
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print("Assets selected for local explanation: {}".format(
test_assets_selected_indexs))
result_store = FileSystemResultStore(result_store_dir)
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict['feature_dict'],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)['ys_label_pred']
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
DisplayConfig.show()
def setUp(self):
ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width': 1920, 'height': 1080})
self.runner = VmafLegacyQualityRunner(
[asset], None, fifo_mode=True,
delete_workdir=True, result_store=FileSystemResultStore(),
)
self.runner.run()
self.result = self.runner.results[0]
def setUp(self):
# ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
# dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
# asset = Asset(dataset="test", content_id=0, asset_id=0,
# workdir_root=VmafConfig.workdir_path(),
# ref_path=ref_path,
# dis_path=dis_path,
# asset_dict={'width':1920, 'height':1080})
#
# self.runner = SsimQualityRunner(
# [asset], None, fifo_mode=True,
# delete_workdir=True, result_store=FileSystemResultStore(),
# )
# self.runner.run()
#
# FileSystemResultStore.save_result(self.runner.results[0], VmafConfig.test_resource_path('/ssim_result_for_test.txt')
self.result = FileSystemResultStore.load_result(VmafConfig.test_resource_path('ssim_result_for_test.txt'))
def setUp(self):
# ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
# dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
# asset = Asset(dataset="test", content_id=0, asset_id=0,
# workdir_root=VmafConfig.workdir_path(),
# ref_path=ref_path,
# dis_path=dis_path,
# asset_dict={'width':1920, 'height':1080})
#
# self.runner = SsimQualityRunner(
# [asset], None, fifo_mode=True,
# delete_workdir=True, result_store=FileSystemResultStore(),
# )
# self.runner.run()
#
# FileSystemResultStore.save_result(self.runner.results[0], VmafConfig.test_resource_path('/ssim_result_for_test.txt')
self.result = FileSystemResultStore.load_result(VmafConfig.test_resource_path('ssim_result_for_test.txt'))
def run_vmaf_cv(train_dataset_filepath,
test_dataset_filepath,
param_filepath,
output_model_filepath=None,
**kwargs):
result_store_dir = kwargs['result_store_dir'] if 'result_store_dir' in kwargs else VmafConfig.file_result_store_path()
logger = get_stdout_logger()
result_store = FileSystemResultStore(result_store_dir)
train_dataset = import_python_file(train_dataset_filepath)
test_dataset = import_python_file(test_dataset_filepath) if test_dataset_filepath is not None else None
param = import_python_file(param_filepath)
# === plot scatter ===
nrows = 1
ncols = 2
fig, axs = plt.subplots(figsize=(5*ncols, 5*nrows), nrows=nrows, ncols=ncols)
train_test_vmaf_on_dataset(train_dataset, test_dataset, param, param, axs[0], axs[1],
result_store, parallelize=True, logger=None,
output_model_filepath=output_model_filepath,
**kwargs)
if 'xlim' in kwargs:
axs[0].set_xlim(kwargs['xlim'])
axs[1].set_xlim(kwargs['xlim'])
if 'ylim' in kwargs:
axs[0].set_ylim(kwargs['ylim'])
axs[1].set_ylim(kwargs['ylim'])
bbox = {'facecolor':'white', 'alpha':1, 'pad':20}
axs[0].annotate('Training Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox)
axs[1].annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox)
plt.tight_layout()
# === clean up ===
close_logger(logger)
def setUp(self):
# ref_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_0_0.yuv"
# dis_path = config.ROOT + "/resource/yuv/checkerboard_1920_1080_10_3_1_0.yuv"
# asset = Asset(dataset="test", content_id=0, asset_id=0,
# workdir_root=config.ROOT + "/workspace/workdir",
# ref_path=ref_path,
# dis_path=dis_path,
# asset_dict={'width':1920, 'height':1080})
#
# self.runner = SsimQualityRunner(
# [asset], None, fifo_mode=True,
# delete_workdir=True, result_store=FileSystemResultStore(),
# )
# self.runner.run()
#
# FileSystemResultStore.save_result(self.runner.results[0], config.ROOT + '/python/test/resource/ssim_result_for_test.txt')
self.result = FileSystemResultStore.load_result(
config.ROOT + '/python/test/resource/ssim_result_for_test.txt')
def run_vmafossexec_with_subsample(dataset_filepath, subsample):
result_store = FileSystemResultStore()
runner_class = VmafossExecQualityRunner
dataset = import_python_file(dataset_filepath)
start_time = time()
assets, results = run_test_on_dataset(dataset, runner_class, None, result_store,
model_filepath=None, parallelize=False,
fifo_mode=True, subsample=subsample)
elapsed_time = time() - start_time
groundtruths = map(lambda asset: asset.groundtruth, assets)
predictions = map(lambda result: result[runner_class.get_score_key()], results)
stats = RegressorMixin.get_stats(groundtruths, predictions)
srcc = stats['SRCC']
pcc = stats['PCC']
rmse = stats['RMSE']
return elapsed_time, srcc, pcc, rmse
def setUp(self):
ref_path = VmafConfig.test_resource_path(
"yuv", "checkerboard_1920_1080_10_3_0_0.yuv")
dis_path = VmafConfig.test_resource_path(
"yuv", "checkerboard_1920_1080_10_3_1_0.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 1920,
'height': 1080
})
self.runner = VmafQualityRunner(
[asset],
None,
fifo_mode=True,
delete_workdir=True,
result_store=FileSystemResultStore(),
optional_dict={
'model_filepath':
VmafConfig.model_path("vmaf_float_v0.6.1.json"),
},
)
self.runner.run()
self.result = self.runner.results[0]
Nframes = len(self.result.result_dict['VMAF_scores'])
self.result.result_dict['VMAF_array_scores'] = self.result.result_dict[
'VMAF_scores'].reshape(Nframes, 1)
self.result.result_dict['VMAF_two_models_array_scores'] = np.vstack(
(self.result.result_dict['VMAF_scores'].reshape(1, Nframes),
self.result.result_dict['VMAF_scores'].reshape(1, Nframes)))
self.result.result_dict['VMAF_3D_array_scores'] = np.zeros((1, 1, 1))
def run_vmaf_kfold_cv(dataset_filepath,
contentid_groups,
param_filepath,
aggregate_method,
):
logger = get_stdout_logger()
result_store = FileSystemResultStore()
dataset = import_python_file(dataset_filepath)
param = import_python_file(param_filepath)
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
cv_on_dataset(dataset, param, param, ax, result_store, contentid_groups,
logger, aggregate_method)
ax.set_xlim([0, 120])
ax.set_ylim([0, 120])
plt.tight_layout()
# === clean up ===
close_logger(logger)
def test_load_result_with_none(self):
result = FileSystemResultStore.load_result(VmafConfig.test_resource_path('result_with_none.txt'))
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(result['STRRED_feature_srred_score'], 5829.2644469999996, places=4)
def test_run_parallel_vmaf_fextractor_with_result_store(self):
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
result_store = FileSystemResultStore(logger=None)
self.fextractor = VmafFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=result_store
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
self.fextractor = VmafFeatureExtractor(
[asset, asset_original, asset],
None, fifo_mode=True,
result_store=result_store
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9345149030293786, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 371.80645372916666, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.90791933424090698, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.8938705209242691, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.9300123587874962, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.9649663148179196, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.9241841443734412, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
self.assertAlmostEqual(results[2]['VMAF_feature_vif_score'], 0.4460930625, places=4)
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractor = VmafFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=result_store
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.fextractor = VmafFeatureExtractor(
[asset, asset_original, asset],
None, fifo_mode=True,
result_store=result_store
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.93458780728708746, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 371.83541406249998, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.90791933424090698, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.89395660507453423, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.93010045185439161, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96503534602850938, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.92425293429958566, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
self.assertAlmostEqual(results[2]['VMAF_feature_vif_score'], 0.4460930625, places=4)
def test_load_result_with_none(self):
print 'test on file system result store load result with None...'
result = FileSystemResultStore.load_result(VmafConfig.test_resource_path('result_with_none.txt'))
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(result['STRRED_feature_srred_score'], 5829.2644469999996, places=4)
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
test_dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv),
'--vmaf-model')
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--parallelize')
print_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--print-result')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--suppress-plot')
vmaf_phone_model = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--vmaf-phone-model')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print '--pool can only have option among {}'.format(
', '.join(POOL_METHODS))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
from sureal.subjective_model import SubjectiveModel
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print "Error: " + str(e)
return 1
save_plot_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--save-plot')
try:
runner_class = QualityRunner.find_subclass(quality_type)
except Exception as e:
print "Error: " + str(e)
return 1
if vmaf_model_path is not None and runner_class != VmafQualityRunner and \
runner_class != BootstrapVmafQualityRunner:
print "Input error: only quality_type of VMAF accepts --vmaf-model."
print_usage()
return 2
if vmaf_phone_model and runner_class != VmafQualityRunner and \
runner_class != BootstrapVmafQualityRunner:
print "Input error: only quality_type of VMAF accepts --vmaf-phone-model."
print_usage()
return 2
try:
test_dataset = import_python_file(test_dataset_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
if vmaf_phone_model:
enable_transform_score = True
else:
enable_transform_score = None
try:
if suppress_plot:
raise AssertionError
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
ax,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score)
bbox = {'facecolor': 'white', 'alpha': 0.5, 'pad': 20}
ax.annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
except ImportError:
print_matplotlib_warning()
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score)
except AssertionError:
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score)
if print_result:
for result in results:
print result
print ''
return 0
def test_run_strred_fextractor_blackframes(self):
print 'test on running STRRED feature extractor on flat frames...'
ref_path = config.ROOT + "/resource/yuv/flat_1920_1080_0.yuv"
dis_path = config.ROOT + "/resource/yuv/flat_1920_1080_10.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
from vmaf.core.result_store import FileSystemResultStore
result_store = FileSystemResultStore(logger=None)
self.fextractor = StrredFeatureExtractor([asset, asset_original],
None,
fifo_mode=True,
result_store=result_store)
print ' running for the first time with fresh calculation...'
self.fextractor.run(parallelize=True)
result0, result1 = self.fextractor.results
import os
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.fextractor.run(parallelize=True)
results = self.fextractor.results
# ignore NaN
for result in results:
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'],
5829.2644469999996,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'],
13086.862734,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'],
62207779.127545856,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'],
0.0,
places=4)
def test_run_strred_fextractor_blackframes(self):
ref_path = VmafConfig.test_resource_path("yuv", "flat_1920_1080_0.yuv")
dis_path = VmafConfig.test_resource_path("yuv",
"flat_1920_1080_10.yuv")
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
from vmaf.core.result_store import FileSystemResultStore
result_store = FileSystemResultStore(logger=None)
self.fextractor = StrredFeatureExtractor([asset, asset_original],
None,
fifo_mode=True,
result_store=result_store)
self.fextractor.run(parallelize=True)
result0, result1 = self.fextractor.results
import os
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(
os.path.exists(result_store._get_result_file_path(result1)))
self.fextractor.run(parallelize=True)
results = self.fextractor.results
# ignore NaN
for result in results:
result.set_score_aggregate_method(ListStats.nonemean)
self.assertAlmostEqual(results[0]['STRRED_feature_srred_score'],
1220.5679849999999,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_trred_score'],
50983.3097155,
places=4)
self.assertAlmostEqual(results[0]['STRRED_feature_strred_score'],
62228595.6081,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_srred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_trred_score'],
0.0,
places=4)
self.assertAlmostEqual(results[1]['STRRED_feature_strred_score'],
0.0,
places=4)
def setUp(self):
super().setUp()
self.result_store = FileSystemResultStore()
def main():
if len(sys.argv) < 5:
print_usage()
return 2
try:
train_dataset_filepath = sys.argv[1]
feature_param_filepath = sys.argv[2]
model_param_filepath = sys.argv[3]
output_model_filepath = sys.argv[4]
except ValueError:
print_usage()
return 2
try:
train_dataset = import_python_file(train_dataset_filepath)
feature_param = import_python_file(feature_param_filepath)
model_param = import_python_file(model_param_filepath)
except Exception as e:
print("Error: %s" % e)
return 1
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--parallelize')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--suppress-plot')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print('--pool can only have option among {}'.format(
', '.join(POOL_METHODS)))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
from sureal.subjective_model import SubjectiveModel
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print("Error: %s" % e)
return 1
save_plot_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--save-plot')
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
logger = None
try:
if suppress_plot:
raise AssertionError
from vmaf import plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=ax,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
bbox = {'facecolor': 'white', 'alpha': 0.5, 'pad': 20}
ax.annotate('Training Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
except ImportError:
print_matplotlib_warning()
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
except AssertionError:
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
return 0
def test_run_parallel_vamf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store ' \
'in parallel...'
ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv")
dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv")
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=VmafConfig.workdir_path(),
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
result_store = FileSystemResultStore(logger=None)
print ' running for the first time with fresh calculation...'
self.fextractor = VmafFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=result_store
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
result0, result1 = results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.fextractor = VmafFeatureExtractor(
[asset, asset_original, asset],
None, fifo_mode=True,
result_store=result_store
)
self.fextractor.run(parallelize=True)
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.93458780728708746, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 371.83541406249998, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.90791933424090698, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.89395660507453423, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.93010045185439161, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96503534602850938, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.92425293429958566, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)
self.assertAlmostEqual(results[2]['VMAF_feature_vif_score'], 0.4460930625, places=4)
def setUp(self):
self.result_store = FileSystemResultStore()
def test_run_vmaf_fextractor_with_result_store(self):
print 'test on running VMAF feature extractor with result store...'
ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing()
result_store = FileSystemResultStore(logger=None)
self.fextractor = VmafFeatureExtractor(
[asset, asset_original],
None, fifo_mode=True,
result_store=result_store
)
print ' running for the first time with fresh calculation...'
self.fextractor.run()
result0, result1 = self.fextractor.results
self.assertTrue(os.path.exists(result_store._get_result_file_path(result0)))
self.assertTrue(os.path.exists(result_store._get_result_file_path(result1)))
print ' running for the second time with stored results...'
self.fextractor.run()
results = self.fextractor.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.4460930625, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],0.93458780728708746, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.5095715208, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_num_score'], 712650.023478, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_num_score'], 371.83541406249998, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_anpsnr_score'], 34.164776875, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale0_score'], 0.363420489439, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale1_score'], 0.766647542135, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale2_score'], 0.862854666902, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif_scale3_score'], 0.915971778036, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale0_score'], 0.90791933424090698, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale1_score'], 0.89395660507453423, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale2_score'], 0.93010045185439161, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm_scale3_score'], 0.96503534602850938, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_vif2_score'], 0.72722361912801026, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm3_score'], 0.92425293429958566, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.04982535417, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion2_score'], 3.8953518541666665, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.2714392708, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_num_score'], 1597314.86733, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_den_score'], 1597314.95249, places=0)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_num_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_den_score'], 397.83378972916671, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_anpsnr_score'], 41.9266444375, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale0_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale1_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm_scale3_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm3_score'], 1.0, places=4)