def run_vmaf_cv(train_dataset_filepath,
test_dataset_filepath,
param_filepath,
output_model_filepath=None,
**kwargs):
logger = get_stdout_logger()
result_store = FileSystemResultStore()
train_dataset = import_python_file(train_dataset_filepath)
test_dataset = import_python_file(
test_dataset_filepath) if test_dataset_filepath is not None else None
param = import_python_file(param_filepath)
# === plot scatter ===
nrows = 1
ncols = 2
fig, axs = plt.subplots(figsize=(5 * ncols, 5 * nrows),
nrows=nrows,
ncols=ncols)
train_test_vmaf_on_dataset(train_dataset,
test_dataset,
param,
param,
axs[0],
axs[1],
result_store,
parallelize=True,
logger=None,
output_model_filepath=output_model_filepath,
**kwargs)
if 'xlim' in kwargs:
axs[0].set_xlim(kwargs['xlim'])
axs[1].set_xlim(kwargs['xlim'])
if 'ylim' in kwargs:
axs[0].set_ylim(kwargs['ylim'])
axs[1].set_ylim(kwargs['ylim'])
bbox = {'facecolor': 'white', 'alpha': 1, 'pad': 20}
axs[0].annotate('Training Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
axs[1].annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
plt.tight_layout()
# === clean up ===
close_logger(logger)
def run_vmaf_kfold_cv(
dataset_filepath,
contentid_groups,
param_filepath,
aggregate_method,
):
logger = get_stdout_logger()
result_store = FileSystemResultStore()
dataset = import_python_file(dataset_filepath)
param = import_python_file(param_filepath)
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
cv_on_dataset(dataset, param, param, ax, result_store, contentid_groups,
logger, aggregate_method)
ax.set_xlim([0, 120])
ax.set_ylim([0, 120])
plt.tight_layout()
# === clean up ===
close_logger(logger)
def main():
subsamples = [1, 2, 4, 8, 16, 32, 64, 128]
elapsed_times = []
pccs = []
for subsample in subsamples:
elapsed_time, srcc, pcc, rmse = run_vmafossexec_with_subsample(
VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py'),
subsample)
elapsed_times.append(elapsed_time)
pccs.append(pcc)
print("SRCC: {}, PCC: {}, RMSE: {}, time: {}".format(
srcc, pcc, rmse, elapsed_time))
fig, ax = plt.subplots(1, 1, figsize=[8, 5])
ax.plot(subsamples, 6 * 24 * 79 / np.array(elapsed_times), 'x-')
ax.set_xlabel("Subsample")
ax.set_ylabel("Processing Speed (Frms/Sec)")
ax.grid(True)
plt.tight_layout()
DisplayConfig.show()
def main():
if len(sys.argv) < 5:
print_usage()
return 2
try:
train_dataset_filepath = sys.argv[1]
feature_param_filepath = sys.argv[2]
model_param_filepath = sys.argv[3]
output_model_filepath = sys.argv[4]
except ValueError:
print_usage()
return 2
try:
train_dataset = import_python_file(train_dataset_filepath)
feature_param = import_python_file(feature_param_filepath)
model_param = import_python_file(model_param_filepath)
except Exception as e:
print("Error: %s" % e)
return 1
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--parallelize')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--suppress-plot')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print('--pool can only have option among {}'.format(
', '.join(POOL_METHODS)))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
if subj_model is not None:
from sureal.subjective_model import SubjectiveModel
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = None
except Exception as e:
print("Error: %s" % e)
return 1
save_plot_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--save-plot')
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
logger = None
try:
if suppress_plot:
raise AssertionError
from vmaf import plt
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=ax,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
bbox = {'facecolor': 'white', 'alpha': 0.5, 'pad': 20}
ax.annotate('Training Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
except ImportError:
print_matplotlib_warning()
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
except AssertionError:
train_test_vmaf_on_dataset(
train_dataset=train_dataset,
test_dataset=None,
feature_param=feature_param,
model_param=model_param,
train_ax=None,
test_ax=None,
result_store=result_store,
parallelize=parallelize,
logger=logger,
output_model_filepath=output_model_filepath,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
)
return 0
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
quality_type = sys.argv[1]
test_dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
vmaf_model_path = get_cmd_option(sys.argv, 3, len(sys.argv),
'--vmaf-model')
cache_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--cache-result')
parallelize = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--parallelize')
processes = get_cmd_option(sys.argv, 3, len(sys.argv), '--processes')
print_result = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--print-result')
suppress_plot = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--suppress-plot')
vmaf_phone_model = cmd_option_exists(sys.argv, 3, len(sys.argv),
'--vmaf-phone-model')
pool_method = get_cmd_option(sys.argv, 3, len(sys.argv), '--pool')
if not (pool_method is None or pool_method in POOL_METHODS):
print('--pool can only have option among {}'.format(
', '.join(POOL_METHODS)))
return 2
subj_model = get_cmd_option(sys.argv, 3, len(sys.argv), '--subj-model')
try:
from sureal.subjective_model import SubjectiveModel
if subj_model is not None:
subj_model_class = SubjectiveModel.find_subclass(subj_model)
else:
subj_model_class = SubjectiveModel.find_subclass('MLE_CO_AP2')
except Exception as e:
print("Error: " + str(e))
return 1
save_plot_dir = get_cmd_option(sys.argv, 3, len(sys.argv), '--save-plot')
plot_wh = get_cmd_option(sys.argv, 3, len(sys.argv), '--plot-wh')
if plot_wh is not None:
try:
mo = re.match(r"([0-9]+)x([0-9]+)", plot_wh)
assert mo is not None
w = mo.group(1)
h = mo.group(2)
w = int(w)
h = int(h)
plot_wh = (w, h)
except Exception as e:
print("Error: plot_wh must be in the format of WxH, example: 5x5")
return 1
try:
runner_class = QualityRunner.find_subclass(quality_type)
except Exception as e:
print("Error: " + str(e))
return 1
if vmaf_model_path is not None and runner_class != VmafQualityRunner and \
runner_class != BootstrapVmafQualityRunner:
print("Input error: only quality_type of VMAF accepts --vmaf-model.")
print_usage()
return 2
if vmaf_phone_model and runner_class != VmafQualityRunner and \
runner_class != BootstrapVmafQualityRunner:
print(
"Input error: only quality_type of VMAF accepts --vmaf-phone-model."
)
print_usage()
return 2
if processes is not None:
try:
processes = int(processes)
except ValueError:
print("Input error: processes must be an integer")
assert processes >= 1
try:
test_dataset = import_python_file(test_dataset_filepath)
except Exception as e:
print("Error: " + str(e))
return 1
if cache_result:
result_store = FileSystemResultStore()
else:
result_store = None
# pooling
if pool_method == 'harmonic_mean':
aggregate_method = ListStats.harmonic_mean
elif pool_method == 'min':
aggregate_method = np.min
elif pool_method == 'median':
aggregate_method = np.median
elif pool_method == 'perc5':
aggregate_method = ListStats.perc5
elif pool_method == 'perc10':
aggregate_method = ListStats.perc10
elif pool_method == 'perc20':
aggregate_method = ListStats.perc20
else: # None or 'mean'
aggregate_method = np.mean
if vmaf_phone_model:
enable_transform_score = True
else:
enable_transform_score = None
try:
if suppress_plot:
raise AssertionError
from vmaf import plt
if plot_wh is None:
plot_wh = (5, 5)
fig, ax = plt.subplots(figsize=plot_wh, nrows=1, ncols=1)
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
ax,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score,
processes=processes,
)
bbox = {'facecolor': 'white', 'alpha': 0.5, 'pad': 20}
ax.annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
# ax.set_xlim([-10, 110])
# ax.set_ylim([-10, 110])
plt.tight_layout()
if save_plot_dir is None:
DisplayConfig.show()
else:
DisplayConfig.show(write_to_dir=save_plot_dir)
except ImportError:
print_matplotlib_warning()
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score,
processes=processes,
)
except AssertionError:
assets, results = run_test_on_dataset(
test_dataset,
runner_class,
None,
result_store,
vmaf_model_path,
parallelize=parallelize,
aggregate_method=aggregate_method,
subj_model_class=subj_model_class,
enable_transform_score=enable_transform_score,
processes=processes,
)
if print_result:
for result in results:
print(result)
print('')
return 0
def plot_explanations(cls, exps, assets=None, ys=None, ys_pred=None):
# asserts
N = cls.assert_explanations(exps, assets, ys, ys_pred)
figs = []
for n in range(N):
weights = exps['feature_weights'][n]
features = exps['features'][n]
normalized = exps['features_normalized'][n]
asset = assets[n] if assets is not None else None
y = ys['label'][n] if ys is not None else None
y_pred = ys_pred[n] if ys_pred is not None else None
img = None
if asset is not None:
w, h = asset.dis_width_height
with YuvReader(filepath=asset.dis_path,
width=w,
height=h,
yuv_type=asset.dis_yuv_type) as yuv_reader:
for yuv in yuv_reader:
img, _, _ = yuv
break
assert img is not None
title = ""
if asset is not None:
title += "{}\n".format(
get_file_name_without_extension(asset.ref_path))
if y is not None:
title += "ground truth: {:.3f}\n".format(y)
if y_pred is not None:
title += "predicted: {:.3f}\n".format(y_pred)
if title != "" and title[-1] == '\n':
title = title[:-1]
assert len(weights) == len(features)
M = len(weights)
fig = plt.figure()
ax_top = plt.subplot(2, 1, 1)
ax_left = plt.subplot(2, 3, 4)
ax_mid = plt.subplot(2, 3, 5, sharey=ax_left)
ax_right = plt.subplot(2, 3, 6, sharey=ax_left)
if img is not None:
ax_top.imshow(img, cmap='Greys_r')
ax_top.get_xaxis().set_visible(False)
ax_top.get_yaxis().set_visible(False)
ax_top.set_title(title)
pos = np.arange(M) + 0.1
ax_left.barh(pos, features, color='b', label='feature')
ax_left.set_xticks(np.arange(0, 1.1, 0.2))
ax_left.set_yticks(pos + 0.35)
ax_left.set_yticklabels(exps['feature_names'])
ax_left.set_title('feature')
ax_mid.barh(pos, normalized, color='g', label='fnormal')
ax_mid.get_yaxis().set_visible(False)
ax_mid.set_title('fnormal')
ax_right.barh(pos, weights, color='r', label='weight')
ax_right.get_yaxis().set_visible(False)
ax_right.set_title('weight')
plt.tight_layout()
figs.append(fig)
return figs
