def _save(self, path):
"""Save the ErrorAnalysisManager to the given path.
:param path: The directory path to save the ErrorAnalysisManager to.
:type path: str
"""
top_dir = Path(path)
top_dir.mkdir(parents=True, exist_ok=True)
if len(self._ea_config_list) != len(self._ea_report_list):
raise ConfigAndResultMismatchException(
"The number of error analysis configs {0} doesn't match the "
"number of results {1}".format(len(self._ea_config_list),
len(self._ea_report_list)))
for index in range(0, len(self._ea_report_list)):
# save the configs
directory_manager = DirectoryManager(parent_directory_path=path)
config_path = (directory_manager.create_config_directory() /
'config.json')
ea_config = self._ea_config_list[index]
with open(config_path, 'w') as file:
json.dump(ea_config, file, default=config_json_converter)
# save the reports
report_path = (directory_manager.create_data_directory() /
'report.json')
ea_report = self._ea_report_list[index]
with open(report_path, 'w') as file:
json.dump(ea_report, file, default=report_json_converter)
def validate_state_directory(path, manager_type):
all_dirs = os.listdir(path)
assert manager_type in all_dirs
all_component_paths = os.listdir(path / manager_type)
for component_path in all_component_paths:
# Test if the component directory has UUID structure
UUID(component_path, version=4)
dm = DirectoryManager(path / manager_type, component_path)
config_path = dm.get_config_directory()
data_path = dm.get_data_directory()
generators_path = dm.get_generators_directory()
if manager_type == ManagerNames.EXPLAINER:
assert not config_path.exists()
assert data_path.exists()
assert not generators_path.exists()
elif manager_type == ManagerNames.COUNTERFACTUAL:
assert config_path.exists()
assert data_path.exists()
assert generators_path.exists()
elif manager_type == ManagerNames.ERROR_ANALYSIS:
assert config_path.exists()
assert data_path.exists()
assert not generators_path.exists()
elif manager_type == ManagerNames.CAUSAL:
assert not config_path.exists()
assert data_path.exists()
assert not generators_path.exists()
def _load(path, rai_insights):
"""Load the CausalManager from the given path.
:param path: The directory path to load the CausalManager from.
:type path: str
:param rai_insights: The loaded parent RAIInsights.
:type rai_insights: RAIInsights
:return: The CausalManager manager after loading.
:rtype: CausalManager
"""
inst = CausalManager.__new__(CausalManager)
# Rehydrate results
all_causal_dirs = DirectoryManager.list_sub_directories(path)
inst.__dict__['_results'] = []
for causal_dir in all_causal_dirs:
dm = DirectoryManager(parent_directory_path=path,
sub_directory_name=causal_dir)
causal_result = CausalResult.load(dm.get_data_directory())
inst.__dict__['_results'].append(causal_result)
# Rehydrate model analysis data
inst.__dict__['_train'] = rai_insights.train
inst.__dict__['_test'] = rai_insights.test
inst.__dict__['_target_column'] = rai_insights.target_column
inst.__dict__['_task_type'] = rai_insights.task_type
inst.__dict__['_categorical_features'] = \
rai_insights.categorical_features
return inst
def _save(self, path):
"""
Save the DataBalanceManager to the given path.
:param path: The directory path to save the DataBalanceManager to.
:type path: str
"""
top_dir = Path(path)
top_dir.mkdir(parents=True, exist_ok=True)
# If manager is not added to RAIInsights, don't save any of its configs
if not self._is_added:
return
dir_manager = DirectoryManager(parent_directory_path=path)
config_dir = dir_manager.create_config_directory()
manager_path = config_dir / MANAGER_JSON
with open(manager_path, "w") as f:
json.dump(self.list(), f)
# if measures have been computed, save the computed measures
if self._data_balance_measures:
measures_path = config_dir / MEASURES_JSON
with open(measures_path, "w") as f:
json.dump(self._data_balance_measures, f)
data_path = dir_manager.create_data_directory() / DATA_JSON
self._df.to_json(data_path, orient="split")
def test_counterfactual_manager_save_load(self, tmpdir):
X_train, X_test, y_train, y_test, feature_names, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
rai_insights = RAIInsights(
model=model,
train=X_train,
test=X_test.iloc[0:10],
target_column='target',
task_type='classification')
rai_insights.counterfactual.add(
total_CFs=10, desired_class=2,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.add(
total_CFs=10, desired_class=1,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.compute()
assert len(rai_insights.counterfactual.get()) == 2
cf_obj = rai_insights.counterfactual.get()[0]
assert cf_obj is not None
save_dir = tmpdir.mkdir('save-dir')
rai_insights.save(save_dir)
rai_insights_copy = RAIInsights.load(save_dir)
assert len(rai_insights_copy.counterfactual.get()) == 2
cf_obj = rai_insights_copy.counterfactual.get()[0]
assert cf_obj is not None
# Delete the dice-ml explainer directory so that the dice-ml
# explainer can be re-trained rather being loaded from the
# disc
counterfactual_path = save_dir / "counterfactual"
all_cf_dirs = DirectoryManager.list_sub_directories(
counterfactual_path)
for counterfactual_config_dir in all_cf_dirs:
directory_manager = DirectoryManager(
parent_directory_path=counterfactual_path,
sub_directory_name=counterfactual_config_dir)
explainer_pkl_path = \
directory_manager.get_generators_directory() / "explainer.pkl"
os.remove(explainer_pkl_path)
rai_insights_copy_new = RAIInsights.load(save_dir)
counterfactual_config_list = \
rai_insights_copy_new.counterfactual._counterfactual_config_list
assert len(counterfactual_config_list) == 2
assert counterfactual_config_list[0].explainer is not None
assert counterfactual_config_list[1].explainer is not None
def _load(path, rai_insights):
"""Load the ErrorAnalysisManager from the given path.
:param path: The directory path to load the ErrorAnalysisManager from.
:type path: str
:param rai_insights: The loaded parent RAIInsights.
:type rai_insights: RAIInsights
:return: The ErrorAnalysisManager manager after loading.
:rtype: ErrorAnalysisManager
"""
# create the ErrorAnalysisManager without any properties using
# the __new__ function, similar to pickle
inst = ErrorAnalysisManager.__new__(ErrorAnalysisManager)
ea_config_list = []
ea_report_list = []
all_ea_dirs = DirectoryManager.list_sub_directories(path)
for ea_dir in all_ea_dirs:
directory_manager = DirectoryManager(parent_directory_path=path,
sub_directory_name=ea_dir)
config_path = (directory_manager.get_config_directory() /
'config.json')
with open(config_path, 'r') as file:
ea_config = json.load(file, object_hook=as_error_config)
ea_config_list.append(ea_config)
report_path = (directory_manager.get_data_directory() /
'report.json')
with open(report_path, 'r') as file:
ea_report = json.load(file, object_hook=as_error_report)
# Validate the serialized output against schema
schema = ErrorAnalysisManager._get_error_analysis_schema()
jsonschema.validate(json.loads(ea_report.to_json()), schema)
ea_report_list.append(ea_report)
inst.__dict__['_ea_report_list'] = ea_report_list
inst.__dict__['_ea_config_list'] = ea_config_list
categorical_features = rai_insights.categorical_features
inst.__dict__['_categorical_features'] = categorical_features
target_column = rai_insights.target_column
true_y = rai_insights.test[target_column]
dataset = rai_insights.test.drop(columns=[target_column])
inst.__dict__['_dataset'] = dataset
inst.__dict__['_true_y'] = true_y
feature_names = list(dataset.columns)
inst.__dict__['_feature_names'] = feature_names
inst.__dict__['_analyzer'] = ModelAnalyzer(rai_insights.model, dataset,
true_y, feature_names,
categorical_features)
return inst
def _load(path, rai_insights):
"""Load the ExplainerManager from the given path.
:param path: The directory path to load the ExplainerManager from.
:type path: str
:param rai_insights: The loaded parent RAIInsights.
:type rai_insights: RAIInsights
:return: The ExplainerManager manager after loading.
:rtype: ExplainerManager
"""
# create the ExplainerManager without any properties using the __new__
# function, similar to pickle
inst = ExplainerManager.__new__(ExplainerManager)
all_cf_dirs = DirectoryManager.list_sub_directories(path)
if len(all_cf_dirs) != 0:
directory_manager = DirectoryManager(
parent_directory_path=path,
sub_directory_name=all_cf_dirs[0])
data_directory = directory_manager.get_data_directory()
with open(data_directory / META_JSON, 'r') as meta_file:
meta = meta_file.read()
meta = json.loads(meta)
inst.__dict__['_' + IS_RUN] = meta[IS_RUN]
inst.__dict__['_' + IS_ADDED] = meta[IS_ADDED]
inst.__dict__[EXPLANATION] = None
explanation_path = data_directory / ManagerNames.EXPLAINER
if explanation_path.exists():
explanation = load_explanation(explanation_path)
inst.__dict__[EXPLANATION] = explanation
else:
inst.__dict__['_' + IS_RUN] = False
inst.__dict__['_' + IS_ADDED] = False
inst.__dict__[EXPLANATION] = None
inst.__dict__['_' + MODEL] = rai_insights.model
inst.__dict__['_' + CLASSES] = rai_insights._classes
inst.__dict__['_' + CATEGORICAL_FEATURES] = \
rai_insights.categorical_features
target_column = rai_insights.target_column
train = rai_insights.train.drop(columns=[target_column])
test = rai_insights.test.drop(columns=[target_column])
inst.__dict__[U_INITIALIZATION_EXAMPLES] = train
inst.__dict__[U_EVALUATION_EXAMPLES] = test
inst.__dict__['_' + FEATURES] = list(train.columns)
# reset the surrogate model
inst._initialize_surrogate_model()
return inst
def _save(self, path):
"""Save the CausalManager to the given path.
:param path: The directory path to save the CausalManager to.
:type path: str
"""
causal_dir = Path(path)
causal_dir.mkdir(parents=True, exist_ok=True)
# Save results to disk
for result in self._results:
directory_manager = DirectoryManager(parent_directory_path=path)
data_path = directory_manager.create_data_directory()
result.save(data_path)
def test_causal_save_and_load(self, housing_data, tmpdir):
train_df, test_df, target_feature = housing_data
save_dir = tmpdir.mkdir('save-dir')
insights = RAIInsights(None, train_df, test_df, target_feature,
ModelTask.REGRESSION)
insights.causal.add(['AveRooms'])
insights.compute()
pre_results = insights.causal.get()
pre_result = pre_results[0]
insights.causal._save(save_dir)
manager = insights.causal._load(save_dir, insights)
post_results = manager.get()
post_result = post_results[0]
assert post_result.id == pre_result.id
assert post_result.causal_analysis is not None
assert post_result.global_effects is not None
assert post_result.local_effects is not None
assert post_result.policies is not None
# Remove the causal analysis models to test the loading of
# causal models in case there is error in loading of the causal
# models.
all_causal_dirs = DirectoryManager.list_sub_directories(save_dir)
for causal_dir in all_causal_dirs:
dm = DirectoryManager(parent_directory_path=save_dir,
sub_directory_name=causal_dir)
causal_analysis_pkl_file_path = \
dm.get_data_directory() / "causal_analysis.pkl"
os.remove(causal_analysis_pkl_file_path)
model_load_err = ('ERROR-LOADING-EXPLAINER: '
'There was an error loading the explainer. '
'Some of RAI dashboard features may not work.')
with pytest.warns(UserWarning, match=model_load_err):
manager = insights.causal._load(save_dir, insights)
post_results = manager.get()
post_result = post_results[0]
assert post_result.id == pre_result.id
assert post_result.causal_analysis is None
assert post_result.global_effects is not None
assert post_result.local_effects is not None
assert post_result.policies is not None
def _save(self, path):
"""Save the CounterfactualManager to the given path.
:param path: The directory path to save the CounterfactualManager to.
:type path: str
"""
counterfactual_dir = Path(path)
counterfactual_dir.mkdir(parents=True, exist_ok=True)
for counterfactual_config in self._counterfactual_config_list:
directory_manager = DirectoryManager(parent_directory_path=path)
counterfactual_config.save_config(
directory_manager.create_config_directory())
counterfactual_config.save_result(
directory_manager.create_data_directory())
counterfactual_config.save_explainer(
directory_manager.create_generators_directory())
def _save(self, path):
"""Save the ExplainerManager to the given path.
:param path: The directory path to save the ExplainerManager to.
:type path: str
"""
top_dir = Path(path)
top_dir.mkdir(parents=True, exist_ok=True)
if self._is_added:
directory_manager = DirectoryManager(parent_directory_path=path)
data_directory = directory_manager.create_data_directory()
# save the explanation
if self._explanation:
save_explanation(self._explanation,
data_directory / ManagerNames.EXPLAINER)
meta = {IS_RUN: self._is_run, IS_ADDED: self._is_added}
with open(data_directory / META_JSON, 'w') as file:
json.dump(meta, file)
def test_directory_manager(self, tmpdir, create_parent_directory):
if create_parent_directory:
parent_directory = tmpdir.mkdir('parent_directory')
else:
parent_directory = tmpdir / 'parent_directory'
dm_one = DirectoryManager(parent_directory_path=parent_directory,
sub_directory_name='known')
assert dm_one.parent_directory_path.exists()
assert dm_one.sub_directory_name == 'known'
assert (dm_one.parent_directory_path /
dm_one.sub_directory_name).exists()
self._verify_directory_manager_operations(dm_one)
assert isinstance(
DirectoryManager.list_sub_directories(parent_directory), list)
assert len(
DirectoryManager.list_sub_directories(parent_directory)) == 1
assert 'known' in\
DirectoryManager.list_sub_directories(parent_directory)
dm_two = DirectoryManager(parent_directory_path=parent_directory)
assert dm_two.parent_directory_path.exists()
assert dm_two.sub_directory_name is not None
assert (dm_two.parent_directory_path /
dm_two.sub_directory_name).exists()
self._verify_directory_manager_operations(dm_two)
assert isinstance(
DirectoryManager.list_sub_directories(parent_directory), list)
assert len(
DirectoryManager.list_sub_directories(parent_directory)) == 2
assert dm_two.sub_directory_name in\
DirectoryManager.list_sub_directories(parent_directory)
def _load(path, rai_insights):
"""Load the CounterfactualManager from the given path.
:param path: The directory path to load the CounterfactualManager from.
:type path: str
:param rai_insights: The loaded parent RAIInsights.
:type rai_insights: RAIInsights
:return: The CounterfactualManager manager after loading.
:rtype: CounterfactualManager
"""
inst = CounterfactualManager.__new__(CounterfactualManager)
# Rehydrate model analysis data
inst.__dict__[CounterfactualManager._MODEL] = rai_insights.model
inst.__dict__[CounterfactualManager._TRAIN] = rai_insights.train
inst.__dict__[CounterfactualManager._TEST] = rai_insights.test
inst.__dict__[CounterfactualManager._TARGET_COLUMN] = \
rai_insights.target_column
inst.__dict__[CounterfactualManager._TASK_TYPE] = \
rai_insights.task_type
inst.__dict__[CounterfactualManager._CATEGORICAL_FEATURES] = \
rai_insights.categorical_features
inst.__dict__[CounterfactualManager._COUNTERFACTUAL_CONFIG_LIST] = []
# DirectoryManager.ensure_dir_exists(path)
all_cf_dirs = DirectoryManager.list_sub_directories(path)
for counterfactual_config_dir in all_cf_dirs:
directory_manager = DirectoryManager(
parent_directory_path=path,
sub_directory_name=counterfactual_config_dir)
counterfactual_config = CounterfactualConfig.load_config(
directory_manager.get_config_directory())
counterfactual_config.load_result(
directory_manager.get_data_directory())
counterfactual_config.load_explainer(
directory_manager.get_generators_directory())
if counterfactual_config.explainer is None:
explainer_load_err = (
'ERROR-LOADING-COUNTERFACTUAL-EXPLAINER: '
'There was an error loading the '
'counterfactual explainer model. '
'Retraining the counterfactual '
'explainer.')
warnings.warn(explainer_load_err)
counterfactual_config.explainer = \
inst._create_diceml_explainer(
counterfactual_config.method,
counterfactual_config.continuous_features)
if counterfactual_config.counterfactual_obj is not None:
# Validate the serialized output against schema
schema = CounterfactualManager._get_counterfactual_schema(
version=counterfactual_config.counterfactual_obj.
metadata['version'])
jsonschema.validate(
json.loads(
counterfactual_config.counterfactual_obj.to_json()),
schema)
inst.__dict__[
CounterfactualManager._COUNTERFACTUAL_CONFIG_LIST].append(
counterfactual_config)
return inst
def test_counterfactual_manager_save_load(self, tmpdir):
X_train, X_test, y_train, y_test, feature_names, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
rai_insights = RAIInsights(model=model,
train=X_train,
test=X_test.iloc[0:10],
target_column='target',
task_type='classification')
rai_insights.counterfactual.add(
total_CFs=10,
desired_class=2,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.add(
total_CFs=10,
desired_class=1,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.compute()
counterfactual_config_list_before_save = \
rai_insights.counterfactual._counterfactual_config_list
assert len(counterfactual_config_list_before_save) == 2
assert len(rai_insights.counterfactual.get()) == 2
cf_obj_1 = rai_insights.counterfactual.get()[0]
assert cf_obj_1 is not None
cf_obj_2 = rai_insights.counterfactual.get()[1]
assert cf_obj_2 is not None
save_dir = tmpdir.mkdir('save-dir')
rai_insights.save(save_dir)
rai_insights_copy = RAIInsights.load(save_dir)
counterfactual_config_list_after_save = \
rai_insights_copy.counterfactual._counterfactual_config_list
assert len(rai_insights_copy.counterfactual.get()) == 2
cf_obj_1 = rai_insights_copy.counterfactual.get()[0]
assert cf_obj_1 is not None
cf_obj_2 = rai_insights_copy.counterfactual.get()[1]
assert cf_obj_2 is not None
assert counterfactual_config_list_before_save[0].id in \
[counterfactual_config_list_after_save[0].id,
counterfactual_config_list_after_save[1].id]
assert counterfactual_config_list_before_save[1].id in \
[counterfactual_config_list_after_save[0].id,
counterfactual_config_list_after_save[1].id]
# Delete the dice-ml explainer directory so that the dice-ml
# explainer can be re-trained rather being loaded from the
# disc
counterfactual_path = save_dir / "counterfactual"
all_cf_dirs = DirectoryManager.list_sub_directories(
counterfactual_path)
for counterfactual_config_dir in all_cf_dirs:
directory_manager = DirectoryManager(
parent_directory_path=counterfactual_path,
sub_directory_name=counterfactual_config_dir)
explainer_pkl_path = \
directory_manager.get_generators_directory() / "explainer.pkl"
os.remove(explainer_pkl_path)
with pytest.warns(UserWarning,
match='ERROR-LOADING-COUNTERFACTUAL-EXPLAINER: '
'There was an error loading the '
'counterfactual explainer model. '
'Retraining the counterfactual '
'explainer.'):
rai_insights_copy_new = RAIInsights.load(save_dir)
counterfactual_config_list = \
rai_insights_copy_new.counterfactual._counterfactual_config_list
assert len(counterfactual_config_list) == 2
assert counterfactual_config_list[0].explainer is not None
assert counterfactual_config_list[1].explainer is not None
def _load(path, rai_insights):
"""
Load the DataBalanceManager from the given path.
:param path: The directory path to load the DataBalanceManager from.
:type path: str
:param rai_insights: The loaded parent RAIInsights.
:type rai_insights: RAIInsights
:return: The DataBalanceManager after loading.
:rtype: DataBalanceManager
"""
# create the DataBalanceManager without any properties using the
# __new__ function, similar to pickle
inst = DataBalanceManager.__new__(DataBalanceManager)
inst.__dict__["_train"] = rai_insights.train
inst.__dict__["_test"] = rai_insights.test
is_added = False
cols_of_interest = None
task_type = rai_insights.task_type
target_column = rai_insights.target_column
classes = (list(map(str, rai_insights._classes))
if rai_insights._classes is not None else [])
df = pd.concat([rai_insights.train, rai_insights.test])
data_balance_measures = None
all_db_dirs = DirectoryManager.list_sub_directories(path)
if len(all_db_dirs) != 0:
dir_manager = DirectoryManager(parent_directory_path=path,
sub_directory_name=all_db_dirs[0])
config_dir = dir_manager.get_config_directory()
# Load manager
with open(config_dir / MANAGER_JSON, "r") as f:
manager_info = json.load(f)
is_added = manager_info[Keys.IS_ADDED]
task_type = manager_info[Keys.TASK_TYPE]
cols_of_interest = manager_info[Keys.COLS_OF_INTEREST]
target_column = manager_info[Keys.TARGET_COLUMN]
classes = manager_info[Keys.CLASSES]
# Load from data json
data_path = dir_manager.get_data_directory() / DATA_JSON
if data_path.exists():
df = pd.read_json(data_path, orient="split")
# Load measures
measures_path = config_dir / MEASURES_JSON
if measures_path.exists():
with open(measures_path, "r") as f:
data_balance_measures = json.load(f)
inst.__dict__["_is_added"] = is_added
inst.__dict__["_task_type"] = task_type
inst.__dict__["_cols_of_interest"] = cols_of_interest
inst.__dict__["_target_column"] = target_column
inst.__dict__["_classes"] = classes
inst.__dict__["_df"] = df
inst.__dict__["_data_balance_measures"] = data_balance_measures
return inst