def validate_state_directory(path, manager_type):
all_dirs = os.listdir(path)
assert manager_type in all_dirs
all_component_paths = os.listdir(path / manager_type)
for component_path in all_component_paths:
# Test if the component directory has UUID structure
UUID(component_path, version=4)
dm = DirectoryManager(path / manager_type, component_path)
config_path = dm.get_config_directory()
data_path = dm.get_data_directory()
generators_path = dm.get_generators_directory()
if manager_type == ManagerNames.EXPLAINER:
assert not config_path.exists()
assert data_path.exists()
assert not generators_path.exists()
elif manager_type == ManagerNames.COUNTERFACTUAL:
assert config_path.exists()
assert data_path.exists()
assert generators_path.exists()
elif manager_type == ManagerNames.ERROR_ANALYSIS:
assert config_path.exists()
assert data_path.exists()
assert not generators_path.exists()
elif manager_type == ManagerNames.CAUSAL:
assert not config_path.exists()
assert data_path.exists()
assert not generators_path.exists()
def test_counterfactual_manager_save_load(self, tmpdir):
X_train, X_test, y_train, y_test, feature_names, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
rai_insights = RAIInsights(
model=model,
train=X_train,
test=X_test.iloc[0:10],
target_column='target',
task_type='classification')
rai_insights.counterfactual.add(
total_CFs=10, desired_class=2,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.add(
total_CFs=10, desired_class=1,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.compute()
assert len(rai_insights.counterfactual.get()) == 2
cf_obj = rai_insights.counterfactual.get()[0]
assert cf_obj is not None
save_dir = tmpdir.mkdir('save-dir')
rai_insights.save(save_dir)
rai_insights_copy = RAIInsights.load(save_dir)
assert len(rai_insights_copy.counterfactual.get()) == 2
cf_obj = rai_insights_copy.counterfactual.get()[0]
assert cf_obj is not None
# Delete the dice-ml explainer directory so that the dice-ml
# explainer can be re-trained rather being loaded from the
# disc
counterfactual_path = save_dir / "counterfactual"
all_cf_dirs = DirectoryManager.list_sub_directories(
counterfactual_path)
for counterfactual_config_dir in all_cf_dirs:
directory_manager = DirectoryManager(
parent_directory_path=counterfactual_path,
sub_directory_name=counterfactual_config_dir)
explainer_pkl_path = \
directory_manager.get_generators_directory() / "explainer.pkl"
os.remove(explainer_pkl_path)
rai_insights_copy_new = RAIInsights.load(save_dir)
counterfactual_config_list = \
rai_insights_copy_new.counterfactual._counterfactual_config_list
assert len(counterfactual_config_list) == 2
assert counterfactual_config_list[0].explainer is not None
assert counterfactual_config_list[1].explainer is not None
def _load(path, rai_insights):
"""Load the CounterfactualManager from the given path.
:param path: The directory path to load the CounterfactualManager from.
:type path: str
:param rai_insights: The loaded parent RAIInsights.
:type rai_insights: RAIInsights
:return: The CounterfactualManager manager after loading.
:rtype: CounterfactualManager
"""
inst = CounterfactualManager.__new__(CounterfactualManager)
# Rehydrate model analysis data
inst.__dict__[CounterfactualManager._MODEL] = rai_insights.model
inst.__dict__[CounterfactualManager._TRAIN] = rai_insights.train
inst.__dict__[CounterfactualManager._TEST] = rai_insights.test
inst.__dict__[CounterfactualManager._TARGET_COLUMN] = \
rai_insights.target_column
inst.__dict__[CounterfactualManager._TASK_TYPE] = \
rai_insights.task_type
inst.__dict__[CounterfactualManager._CATEGORICAL_FEATURES] = \
rai_insights.categorical_features
inst.__dict__[CounterfactualManager._COUNTERFACTUAL_CONFIG_LIST] = []
# DirectoryManager.ensure_dir_exists(path)
all_cf_dirs = DirectoryManager.list_sub_directories(path)
for counterfactual_config_dir in all_cf_dirs:
directory_manager = DirectoryManager(
parent_directory_path=path,
sub_directory_name=counterfactual_config_dir)
counterfactual_config = CounterfactualConfig.load_config(
directory_manager.get_config_directory())
counterfactual_config.load_result(
directory_manager.get_data_directory())
counterfactual_config.load_explainer(
directory_manager.get_generators_directory())
if counterfactual_config.explainer is None:
explainer_load_err = (
'ERROR-LOADING-COUNTERFACTUAL-EXPLAINER: '
'There was an error loading the '
'counterfactual explainer model. '
'Retraining the counterfactual '
'explainer.')
warnings.warn(explainer_load_err)
counterfactual_config.explainer = \
inst._create_diceml_explainer(
counterfactual_config.method,
counterfactual_config.continuous_features)
if counterfactual_config.counterfactual_obj is not None:
# Validate the serialized output against schema
schema = CounterfactualManager._get_counterfactual_schema(
version=counterfactual_config.counterfactual_obj.
metadata['version'])
jsonschema.validate(
json.loads(
counterfactual_config.counterfactual_obj.to_json()),
schema)
inst.__dict__[
CounterfactualManager._COUNTERFACTUAL_CONFIG_LIST].append(
counterfactual_config)
return inst
def test_counterfactual_manager_save_load(self, tmpdir):
X_train, X_test, y_train, y_test, feature_names, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
rai_insights = RAIInsights(model=model,
train=X_train,
test=X_test.iloc[0:10],
target_column='target',
task_type='classification')
rai_insights.counterfactual.add(
total_CFs=10,
desired_class=2,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.add(
total_CFs=10,
desired_class=1,
features_to_vary=[feature_names[0]],
permitted_range={feature_names[0]: [2.0, 5.0]})
rai_insights.counterfactual.compute()
counterfactual_config_list_before_save = \
rai_insights.counterfactual._counterfactual_config_list
assert len(counterfactual_config_list_before_save) == 2
assert len(rai_insights.counterfactual.get()) == 2
cf_obj_1 = rai_insights.counterfactual.get()[0]
assert cf_obj_1 is not None
cf_obj_2 = rai_insights.counterfactual.get()[1]
assert cf_obj_2 is not None
save_dir = tmpdir.mkdir('save-dir')
rai_insights.save(save_dir)
rai_insights_copy = RAIInsights.load(save_dir)
counterfactual_config_list_after_save = \
rai_insights_copy.counterfactual._counterfactual_config_list
assert len(rai_insights_copy.counterfactual.get()) == 2
cf_obj_1 = rai_insights_copy.counterfactual.get()[0]
assert cf_obj_1 is not None
cf_obj_2 = rai_insights_copy.counterfactual.get()[1]
assert cf_obj_2 is not None
assert counterfactual_config_list_before_save[0].id in \
[counterfactual_config_list_after_save[0].id,
counterfactual_config_list_after_save[1].id]
assert counterfactual_config_list_before_save[1].id in \
[counterfactual_config_list_after_save[0].id,
counterfactual_config_list_after_save[1].id]
# Delete the dice-ml explainer directory so that the dice-ml
# explainer can be re-trained rather being loaded from the
# disc
counterfactual_path = save_dir / "counterfactual"
all_cf_dirs = DirectoryManager.list_sub_directories(
counterfactual_path)
for counterfactual_config_dir in all_cf_dirs:
directory_manager = DirectoryManager(
parent_directory_path=counterfactual_path,
sub_directory_name=counterfactual_config_dir)
explainer_pkl_path = \
directory_manager.get_generators_directory() / "explainer.pkl"
os.remove(explainer_pkl_path)
with pytest.warns(UserWarning,
match='ERROR-LOADING-COUNTERFACTUAL-EXPLAINER: '
'There was an error loading the '
'counterfactual explainer model. '
'Retraining the counterfactual '
'explainer.'):
rai_insights_copy_new = RAIInsights.load(save_dir)
counterfactual_config_list = \
rai_insights_copy_new.counterfactual._counterfactual_config_list
assert len(counterfactual_config_list) == 2
assert counterfactual_config_list[0].explainer is not None
assert counterfactual_config_list[1].explainer is not None