def test_test_on_dataset_mle(self):
test_dataset = import_python_file(
config.ROOT + '/python/test/resource/raw_dataset_sample.py')
test_assets, results = test_on_dataset(test_dataset,
VmafQualityRunner,
None,
None,
None,
parallelize=True,
aggregate_method=None,
subj_model_class=MosModel)
self.assertAlmostEqual(results[0]['VMAF_score'],
99.620284242680668,
places=4)
self.assertAlmostEqual(results[1]['VMAF_score'],
28.98459923031756,
places=4)
self.assertAlmostEqual(results[2]['VMAF_score'],
98.860794831997097,
places=4)
self.assertAlmostEqual(results[3]['VMAF_score'],
98.860790186334995,
places=4)
self.assertAlmostEqual(test_assets[0].groundtruth, 100, places=4)
self.assertAlmostEqual(test_assets[1].groundtruth, 50, places=4)
self.assertAlmostEqual(test_assets[2].groundtruth, 90, places=4)
self.assertAlmostEqual(test_assets[3].groundtruth, 80, places=4)
def test_test_on_dataset_raw(self):
test_dataset = import_python_file(
config.ROOT + '/python/test/resource/raw_dataset_sample.py')
test_assets, results = test_on_dataset(test_dataset, VmafQualityRunner, None,
None, None,
parallelize=True,
aggregate_method=None)
self.assertAlmostEqual(results[0]['VMAF_score'], 99.620284242680668, places=4)
self.assertAlmostEqual(results[1]['VMAF_score'], 28.98459923031756, places=4)
self.assertAlmostEqual(results[2]['VMAF_score'], 98.860794831997097, places=4)
self.assertAlmostEqual(results[3]['VMAF_score'], 98.860790186334995, places=4)
self.assertAlmostEqual(test_assets[0].groundtruth, 100, places=4)
self.assertAlmostEqual(test_assets[1].groundtruth, 50, places=4)
self.assertAlmostEqual(test_assets[2].groundtruth, 100, places=4)
self.assertAlmostEqual(test_assets[3].groundtruth, 90, places=4)
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
test_dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv), '--vmaf-model')
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv), '--parallelize')
print_result = cmd_option_exists(sys.argv, 3, len(sys.argv), '--print-result')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv), '--suppress-plot')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None
or pool_method in POOL_METHODS):
print '--pool can only have option among {}'.format(', '.join(POOL_METHODS))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print "Error: " + str(e)
return 1
if vmaf_model_path is not None and quality_type != VmafQualityRunner.TYPE:
print "Input error: only quality_type of VMAF accepts --vmaf-model."
print_usage()
return 2
try:
test_dataset = import_python_file(test_dataset_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
try:
runner_class = QualityRunner.find_subclass(quality_type)
except Exception as e:
print "Error: " + str(e)
return 1
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
try:
if suppress_plot:
raise AssertionError
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
assets, results = test_on_dataset(test_dataset, runner_class, ax,
result_store, vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
bbox = {'facecolor':'white', 'alpha':0.5, 'pad':20}
ax.annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
plt.show()
except ImportError:
print_matplotlib_warning()
assets, results = test_on_dataset(test_dataset, runner_class, None,
result_store, vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
except AssertionError:
assets, results = test_on_dataset(test_dataset, runner_class, None,
result_store, vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
if print_result:
for result in results:
print result
print ''
return 0
except:
print_usage()
exit(2)
if cache_result == 'yes':
result_store = FileSystemResultStore()
elif cache_result == 'no':
result_store = None
else:
print_usage()
exit(2)
try:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
test_on_dataset(test_dataset, runner_class, ax, result_store,
model_filepath, parallelize)
bbox = {'facecolor': 'white', 'alpha': 1, 'pad': 20}
ax.annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
plt.tight_layout()
plt.show()
except ImportError:
print_matplotlib_warning()
test_on_dataset(test_dataset, runner_class, None, result_store,
model_filepath, parallelize)
print 'Done.'
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
test_dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv), "--vmaf-model")
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv), "--cache-result")
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv), "--parallelize")
print_result = cmd_option_exists(sys.argv, 3, len(sys.argv), "--print-result")
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), "--pool")
if not (pool_method is None or pool_method in POOL_METHODS):
print "--pool can only have option among {}".format(", ".join(POOL_METHODS))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), "--subj-model")
try:
if subj_model is not None:
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print "Error: " + str(e)
return 1
if vmaf_model_path is not None and quality_type != VmafQualityRunner.TYPE:
print "Input error: only quality_type of VMAF accepts --vmaf-model."
print_usage()
return 2
try:
test_dataset = import_python_file(test_dataset_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
try:
runner_class = QualityRunner.find_subclass(quality_type)
except Exception as e:
print "Error: " + str(e)
return 1
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == "harmonic_mean":
aggregate_method = ListStats.harmonic_mean
elif pool_method == "min":
aggregate_method = np.min
elif pool_method == "median":
aggregate_method = np.median
elif pool_method == "perc5":
aggregate_method = ListStats.perc5
elif pool_method == "perc10":
aggregate_method = ListStats.perc10
elif pool_method == "perc20":
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
try:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
assets, results = test_on_dataset(
test_dataset,
runner_class,
ax,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
bbox = {"facecolor": "white", "alpha": 0.5, "pad": 20}
ax.annotate("Testing Set", xy=(0.1, 0.85), xycoords="axes fraction", bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
plt.show()
except ImportError:
print_matplotlib_warning()
assets, results = test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
if print_result:
for result in results:
print result
print ""
return 0
except:
print_usage()
exit(2)
if cache_result == 'yes':
result_store = FileSystemResultStore()
elif cache_result == 'no':
result_store = None
else:
print_usage()
exit(2)
try:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
test_on_dataset(test_dataset, runner_class, ax,
result_store, model_filepath, parallelize)
bbox = {'facecolor':'white', 'alpha':1, 'pad':20}
ax.annotate('Testing Set', xy=(0.1, 0.85), xycoords='axes fraction', bbox=bbox)
plt.tight_layout()
plt.show()
except ImportError:
print_matplotlib_warning()
test_on_dataset(test_dataset, runner_class, None,
result_store, model_filepath, parallelize)
print 'Done.'
exit(0)
