def explain_model_on_dataset(
model,
test_assets_selected_indexs,
test_dataset_filepath,
result_store_dir=VmafConfig.file_result_store_path()):
def print_assets(test_assets):
print('\n'.join(
map(
lambda tasset: "Asset {i}: {name}".format(
i=tasset[0],
name=get_file_name_without_extension(tasset[1].dis_path)),
enumerate(test_assets))))
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print("Assets selected for local explanation: {}".format(
test_assets_selected_indexs))
result_store = FileSystemResultStore(result_store_dir)
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict['feature_dict'],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)['ys_label_pred']
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
DisplayConfig.show()
def run_vmaf_kfold_cv(dataset_filepath,
contentid_groups,
param_filepath,
aggregate_method,
result_store_dir=VmafConfig.file_result_store_path(),
):
logger = get_stdout_logger()
result_store = FileSystemResultStore(result_store_dir)
dataset = import_python_file(dataset_filepath)
param = import_python_file(param_filepath)
fig, ax = plt.subplots(figsize=(5, 5), nrows=1, ncols=1)
cv_on_dataset(dataset, param, param, ax, result_store, contentid_groups,
logger, aggregate_method)
ax.set_xlim([0, 120])
ax.set_ylim([0, 120])
plt.tight_layout()
# === clean up ===
close_logger(logger)
def __init__(self,
logger=None,
result_store_dir=VmafConfig.file_result_store_path()):
self.logger = logger
self.result_store_dir = result_store_dir
def run_vmaf_cv(train_dataset_filepath,
test_dataset_filepath,
param_filepath,
output_model_filepath=None,
**kwargs):
result_store_dir = kwargs[
'result_store_dir'] if 'result_store_dir' in kwargs else VmafConfig.file_result_store_path(
)
logger = get_stdout_logger()
result_store = FileSystemResultStore(result_store_dir)
train_dataset = import_python_file(train_dataset_filepath)
test_dataset = import_python_file(
test_dataset_filepath) if test_dataset_filepath is not None else None
param = import_python_file(param_filepath)
# === plot scatter ===
nrows = 1
ncols = 2
fig, axs = plt.subplots(figsize=(5 * ncols, 5 * nrows),
nrows=nrows,
ncols=ncols)
train_test_vmaf_on_dataset(train_dataset,
test_dataset,
param,
param,
axs[0],
axs[1],
result_store,
parallelize=True,
logger=None,
output_model_filepath=output_model_filepath,
**kwargs)
if 'xlim' in kwargs:
axs[0].set_xlim(kwargs['xlim'])
axs[1].set_xlim(kwargs['xlim'])
if 'ylim' in kwargs:
axs[0].set_ylim(kwargs['ylim'])
axs[1].set_ylim(kwargs['ylim'])
bbox = {'facecolor': 'white', 'alpha': 1, 'pad': 20}
axs[0].annotate('Training Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
axs[1].annotate('Testing Set',
xy=(0.1, 0.85),
xycoords='axes fraction',
bbox=bbox)
plt.tight_layout()
# === clean up ===
close_logger(logger)
def __init__(self, logger=None,
result_store_dir=VmafConfig.file_result_store_path()
):
self.logger = logger
self.result_store_dir = result_store_dir
