def test_explain_train_test_model(self):
model_class = SklearnRandomForestTrainTestModel
train_dataset_path = config.ROOT + '/python/test/resource/' \
'test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
_, self.features = run_executors_in_parallel(
MomentNorefFeatureExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
xys = model_class.get_xys_from_results(self.features[:7])
model = model_class({'norm_type':'normalize', 'random_state':0}, None)
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'],
['Moment_noref_feature_1st_score',
'Moment_noref_feature_2nd_score',
'Moment_noref_feature_var_score']
)
def test_explain_train_test_model(self):
model_class = SklearnRandomForestTrainTestModel
train_dataset_path = config.ROOT + '/python/test/resource/' \
'test_image_dataset_diffdim.py'
train_dataset = import_python_file(train_dataset_path)
train_assets = read_dataset(train_dataset)
_, self.features = run_executors_in_parallel(
MomentNorefFeatureExtractor,
train_assets,
fifo_mode=True,
delete_workdir=True,
parallelize=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
)
xys = model_class.get_xys_from_results(self.features[:7])
model = model_class({'norm_type':'normalize', 'random_state':0}, None)
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'],
['Moment_noref_feature_1st_score',
'Moment_noref_feature_2nd_score',
'Moment_noref_feature_var_score']
)
def explain_model_on_dataset(model, test_assets_selected_indexs,
test_dataset_filepath):
def print_assets(test_assets):
print '\n'.join(
map(
lambda (i, asset): "Asset {i}: {name}".format(
i=i, name=get_file_name_without_extension(asset.dis_path)),
enumerate(test_assets)))
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print "Assets selected for local explanation: {}".format(
test_assets_selected_indexs)
result_store = FileSystemResultStore()
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict['feature_dict'],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
plt.show()
def test_explain_train_test_model(self):
model_class = MomentRandomForestTrainTestModel
xys = model_class.get_xys_from_results(self.features[:7])
del xys['dis_u']
del xys['dis_v']
model = model_class({'norm_type':'normalize', 'random_state':0})
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
del xs['dis_u']
del xs['dis_v']
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'], ['dis_y'])
def test_explain_train_test_model(self):
model_class = MomentRandomForestTrainTestModel
xys = model_class.get_xys_from_results(self.features[:7])
del xys['dis_u']
del xys['dis_v']
model = model_class({'norm_type':'normalize', 'random_state':0})
model.train(xys)
np.random.seed(0)
xs = model_class.get_xs_from_results(self.features[7:])
del xs['dis_u']
del xs['dis_v']
explainer = LocalExplainer(neighbor_samples=1000)
exps = explainer.explain(model, xs)
self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4)
self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4)
self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4)
self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4)
self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4)
self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4)
self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4)
self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4)
self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4)
self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4)
self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4)
self.assertEqual(exps['feature_names'], ['dis_y'])
def explain_model_on_dataset(model, test_assets_selected_indexs, test_dataset_filepath):
def print_assets(test_assets):
print "\n".join(
map(
lambda (i, asset): "Asset {i}: {name}".format(
i=i, name=get_file_name_without_extension(asset.dis_path)
),
enumerate(test_assets),
)
)
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print "Assets selected for local explanation: {}".format(test_assets_selected_indexs)
result_store = FileSystemResultStore()
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict["feature_dict"],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred)
plt.show()
def _run_on_asset(self, asset):
# Override VmafQualityRunner._run_on_asset(self, asset), by adding
# additional local explanation info.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
model = self._load_model(asset)
xs = model.get_per_unit_xs_from_a_result(feature_result)
ys_pred = self.predict_with_model(model, xs)
if self.optional_dict2 is not None and \
'explainer' in self.optional_dict2:
explainer = self.optional_dict2['explainer']
else:
explainer = LocalExplainer()
exps = explainer.explain(model, xs)
result_dict = {}
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = ys_pred # add quality score
result_dict[self.get_explanations_key()] = exps # add local explanations
return Result(asset, self.executor_id, result_dict)
def _run_on_asset(self, asset):
# Override VmafQualityRunner._run_on_asset(self, asset), by adding
# additional local explanation info.
vmaf_fassembler = self._get_vmaf_feature_assembler_instance(asset)
vmaf_fassembler.run()
feature_result = vmaf_fassembler.results[0]
model = self._load_model(asset)
xs = model.get_per_unit_xs_from_a_result(feature_result)
ys_pred = self.predict_with_model(model, xs)
if self.optional_dict2 is not None and \
'explainer' in self.optional_dict2:
explainer = self.optional_dict2['explainer']
else:
explainer = LocalExplainer()
exps = explainer.explain(model, xs)
result_dict = {}
result_dict.update(feature_result.result_dict) # add feature result
result_dict[self.get_scores_key()] = ys_pred # add quality score
result_dict[
self.get_explanations_key()] = exps # add local explanations
return Result(asset, self.executor_id, result_dict)
