def test_train_labels(self):
X_train, X_test, y_train, y_test, _, _ = \
create_cancer_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
train_labels=[0, 1, 2])
assert 'The train labels and distinct values in ' + \
'target (train data) do not match' in str(ucve.value)
y_train[0] = 2
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
train_labels=[0, 1])
assert 'The train labels and distinct values in target ' + \
'(train data) do not match' in str(ucve.value)
y_train[0] = 0
y_test[0] = 2
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
train_labels=[0, 1])
assert 'The train labels and distinct values in target ' + \
'(test data) do not match' in str(ucve.value)
def test_mismatch_train_test_features(self):
X_train, X_test, y_train, y_test, _, _ = \
create_cancer_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['bad_target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
assert 'The features in train and test data do not match' in \
str(ucve.value)
def test_validate_bad_target_name(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='bad_target',
task_type='classification')
assert "Target name bad_target not present in train/test data" in \
str(ucve.value)
def test_validate_unsupported_task_type(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='regre')
assert 'Unsupported task type. Should be one of classification or ' + \
'regression' in str(ucve.value)
def test_unsupported_train_test_types(self):
X_train, X_test, y_train, y_test, _, _ = \
create_cancer_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['bad_target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(
model=model,
train=X_train.values,
test=X_test.values,
target_column='target',
task_type='classification')
assert "Unsupported data type for either train or test. " + \
"Expecting pandas DataFrame for train and test." in str(ucve.value)
def test_validate_categorical_features_having_target(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
categorical_features=['target'])
assert 'Found target name target in categorical feature list' in \
str(ucve.value)
def test_validate_unsupported_task_type(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
message = ("Unsupported task type 'regre'. "
"Should be one of \\['classification', 'regression'\\]")
with pytest.raises(UserConfigValidationException, match=message):
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='regre')
def test_validate_categorical_features_not_having_train_features(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
categorical_features=['some bad feature name'])
assert 'Found some feature names in categorical feature which' + \
' do not occur in train data' in str(ucve.value)
def test_model_predictions_predict(self):
X_train, X_test, y_train, y_test, _, _ = \
create_cancer_data()
X_train['target'] = y_train
X_test['target'] = y_test
model = MagicMock()
model.predict.side_effect = Exception()
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
assert 'The model passed cannot be used for getting predictions ' + \
'via predict()' in str(ucve.value)
def test_validate_categorical_features_not_having_train_features(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
message = ("Feature names in categorical_features "
"do not exist in train data: \\['not_a_feature'\\]")
with pytest.raises(UserConfigValidationException, match=message):
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
categorical_features=['not_a_feature'])
def test_model_analysis_incorrect_task_type(self):
X_train, X_test, y_train, y_test, _, _ = \
create_cancer_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
err_msg = ('The regression model'
'provided has a predict_proba function. '
'Please check the task_type.')
with pytest.raises(UserConfigValidationException, match=err_msg):
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='regression')
def test_treatment_features_list_not_having_train_features(self):
X_train, y_train, X_test, y_test, _ = \
create_binary_classification_dataset()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
model_analysis = ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
message = ("Feature names in treatment_features "
"do not exist in train data: \\['not_a_feature'\\]")
with pytest.raises(UserConfigValidationException, match=message):
model_analysis.causal.add(treatment_features=['not_a_feature'])
def test_treatment_features_list_not_having_train_features(self):
X_train, y_train, X_test, y_test, _ = \
create_binary_classification_dataset()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
model_analysis = ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
with pytest.raises(
UserConfigValidationException,
match='Found some feature names in treatment feature list' +
' which do not occur in train data'):
model_analysis.causal.add(treatment_features=['random'])
def test_permitted_range_not_having_train_features(self):
X_train, y_train, X_test, y_test, _ = \
create_binary_classification_dataset()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
model_analysis = ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
message = ("Feature names in permitted_range do "
"not exist in train data: \\['not_a_feature'\\]")
with pytest.raises(UserConfigValidationException, match=message):
model_analysis.counterfactual.add(
total_CFs=10, permitted_range={'not_a_feature': [20, 40]})
def test_validate_test_data_size(self):
X_train, X_test, y_train, y_test, _, _ = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='bad_target',
task_type='classification',
maximum_rows_for_test=len(y_test) - 1)
assert "The test data has 31 rows, but limit is set to 30 rows" in \
str(ucve.value)
assert "Please resample the test data or " +\
"adjust maximum_rows_for_test" in \
str(ucve.value)
def test_desired_class_not_set(self):
X_train, y_train, X_test, y_test, _ = \
create_binary_classification_dataset()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
model_analysis = ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
with pytest.raises(
UserConfigValidationException,
match='The desired_class attribute should be '
'either \'opposite\' for binary classification or '
'the class value for multi-classification scenarios.'):
model_analysis.counterfactual.add(
total_CFs=10,
method='random')
def test_desired_class_opposite_multi_classification(self):
X_train, X_test, y_train, y_test, feature_names, classes = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
model_analysis = ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
with pytest.raises(
UserConfigValidationException,
match='The desired_class attribute should not be \'opposite\''
' It should be the class value for multiclass'
' classification scenario.'):
model_analysis.counterfactual.add(
total_CFs=10,
method='random',
desired_class='opposite')
def test_desired_range_not_set(self):
X_train, X_test, y_train, y_test, feature_names = \
create_housing_data()
model = create_sklearn_random_forest_regressor(X_train, y_train)
X_train = pd.DataFrame(X_train, columns=feature_names)
X_test = pd.DataFrame(X_test, columns=feature_names)
X_train['TARGET'] = y_train
X_test['TARGET'] = y_test
model_analysis = ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='TARGET',
task_type='regression')
with pytest.raises(
UserConfigValidationException,
match='The desired_range should not be None'
' for regression scenarios.'):
model_analysis.counterfactual.add(
total_CFs=10,
method='random')
def test_eval_data_having_new_categories(self):
train_data = pd.DataFrame(
data=[[1, 2, 0],
[2, 3, 1],
[3, 3, 0]],
columns=['c1', 'c2', 'target']
)
test_data = pd.DataFrame(
data=[[1, 1, 0]],
columns=['c1', 'c2', 'target']
)
X_train = train_data.drop(['target'], axis=1)
y_train = train_data['target']
model = create_lightgbm_classifier(X_train, y_train)
model_analysis = ModelAnalysis(
model=model,
train=train_data,
test=test_data,
target_column='target',
task_type='classification',
categorical_features=['c2'])
message = ("Counterfactual example generation requires "
"that every category of "
"categorical features present in the test data be "
"also present in the train data. "
"Categories missing from train data: "
"{'c2': \\[1\\]}")
with pytest.raises(
UserConfigValidationException, match=message):
model_analysis.counterfactual.add(
total_CFs=10,
method='random',
desired_class='opposite')
def test_feature_importance_with_less_counterfactuals(self):
X_train, X_test, y_train, y_test, feature_names, classes = \
create_iris_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
model_analysis = ModelAnalysis(
model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification')
with pytest.raises(
UserConfigValidationException,
match="A total_CFs value of at least 10 is required to "
"use counterfactual feature importances. "
"Either increase total_CFs to at least 10 or "
"set feature_importance to False."):
model_analysis.counterfactual.add(
total_CFs=5,
method='random',
desired_class=2)
def test_validate_serializer(self):
X_train, X_test, y_train, y_test, _, _ = \
create_cancer_data()
model = create_lightgbm_classifier(X_train, y_train)
X_train['target'] = y_train
X_test['target'] = y_test
with pytest.raises(UserConfigValidationException) as ucve:
class LoadOnlySerializer:
def __init__(self, logger=None):
self._logger = logger
def load(self):
pass
serializer = LoadOnlySerializer()
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
serializer=serializer)
assert 'The serializer does not implement save()' in str(ucve.value)
with pytest.raises(UserConfigValidationException) as ucve:
class SaveOnlySerializer:
def __init__(self, logger=None):
self._logger = logger
def save(self):
pass
serializer = SaveOnlySerializer()
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
serializer=serializer)
assert 'The serializer does not implement load()' in str(ucve.value)
with pytest.raises(UserConfigValidationException) as ucve:
class Serializer:
def __init__(self, logger=None):
self._logger = logger
def save(self):
pass
def load(self):
pass
serializer = Serializer(logger=logging.getLogger('some logger'))
ModelAnalysis(model=model,
train=X_train,
test=X_test,
target_column='target',
task_type='classification',
serializer=serializer)
assert 'The serializer should be serializable via pickle' in \
str(ucve.value)