def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
subjective_model = sys.argv[1]
dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
output_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--output-dir')
print_ = cmd_option_exists(sys.argv, 3, len(sys.argv), '--print')
do_plot = ['raw_scores', 'quality_scores']
if subjective_model in ['MLE', 'MLE_CO', 'DMOS_MLE', 'DMOS_MLE_CO']:
do_plot.append('subject_scores')
if subjective_model in ['MLE', 'DMOS_MLE']:
do_plot.append('content_scores')
try:
subjective_model_class = SubjectiveModel.find_subclass(subjective_model)
except Exception as e:
print "Error: " + str(e)
return 1
print "Run model {} on dataset {}".format(
subjective_model_class.__name__, get_file_name_with_extension(dataset_filepath)
)
dataset, subjective_models, results = run_subjective_models(
dataset_filepath=dataset_filepath,
subjective_model_classes = [subjective_model_class,],
normalize_final=False, # True or False
do_plot=do_plot,
plot_type='errorbar',
gradient_method='simplified',
)
if print_:
print("Dataset: {}".format(dataset_filepath))
print("Subjective Model: {} {}".format(subjective_models[0].TYPE, subjective_models[0].VERSION))
print("Result:")
json.dumps(results[0], indent=4, sort_keys=True)
if output_dir is None:
DisplayConfig.show()
else:
print("Output wrote to {}.".format(output_dir))
DisplayConfig.show(write_to_dir=output_dir)
with open(os.path.join(output_dir, 'sureal.json'), 'w') as out_f:
json.dump(results[0], out_f, indent=4, sort_keys=True)
return 0
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
test_dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv),
'--vmaf-model')
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--parallelize')
print_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--print-result')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--suppress-plot')
vmaf_phone_model = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--vmaf-phone-model')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print '--pool can only have option among {}'.format(
', '.join(POOL_METHODS))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
from sureal.subjective_model import SubjectiveModel
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print "Error: " + str(e)
return 1
save_plot_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--save-plot')
try:
runner_class = QualityRunner.find_subclass(quality_type)
except Exception as e:
print "Error: " + str(e)
return 1
if vmaf_model_path is not None and runner_class != VmafQualityRunner and \
runner_class != BootstrapVmafQualityRunner:
print "Input error: only quality_type of VMAF accepts --vmaf-model."
print_usage()
return 2
if vmaf_phone_model and runner_class != VmafQualityRunner and \
runner_class != BootstrapVmafQualityRunner:
print "Input error: only quality_type of VMAF accepts --vmaf-phone-model."
print_usage()
return 2
try:
test_dataset = import_python_file(test_dataset_filepath)
except Exception as e:
print "Error: " + str(e)
return 1
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
if vmaf_phone_model:
enable_transform_score = True
else:
enable_transform_score = None
try:
if suppress_plot:
raise AssertionError
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
ax,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score)
bbox = {'facecolor': 'white', 'alpha': 0.5, 'pad': 20}
ax.annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
except ImportError:
print_matplotlib_warning()
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score)
except AssertionError:
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score)
if print_result:
for result in results:
print result
print ''
return 0
def main():
if len(sys.argv) < 5:
print_usage()
return 2
try:
train_dataset_filepath = sys.argv[1]
feature_param_filepath = sys.argv[2]
model_param_filepath = sys.argv[3]
output_model_filepath = sys.argv[4]
except ValueError:
print_usage()
return 2
try:
train_dataset = import_python_file(train_dataset_filepath)
feature_param = import_python_file(feature_param_filepath)
model_param = import_python_file(model_param_filepath)
except Exception as e:
print("Error: %s" % e)
return 1
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--parallelize')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--suppress-plot')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print('--pool can only have option among {}'.format(
', '.join(POOL_METHODS)))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
from sureal.subjective_model import SubjectiveModel
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print("Error: %s" % e)
return 1
save_plot_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--save-plot')
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
logger = None
try:
if suppress_plot:
raise AssertionError
from vmaf import plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=ax,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
bbox = {'facecolor': 'white', 'alpha': 0.5, 'pad': 20}
ax.annotate('Training Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
except ImportError:
print_matplotlib_warning()
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
except AssertionError:
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
return 0