def test_shap_config():
baseline = [[
0.26124998927116394,
0.2824999988079071,
0.06875000149011612,
]]
num_samples = 100
agg_method = "mean_sq"
use_logit = True
seed = 123
shap_config = SHAPConfig(
baseline=baseline,
num_samples=num_samples,
agg_method=agg_method,
use_logit=use_logit,
seed=seed,
)
expected_config = {
"shap": {
"baseline": baseline,
"num_samples": num_samples,
"agg_method": agg_method,
"use_logit": use_logit,
"save_local_shap_values": True,
"seed": seed,
}
}
assert expected_config == shap_config.get_explainability_config()
def test_shap_config_no_parameters():
shap_config = SHAPConfig()
expected_config = {
"shap": {
"use_logit": False,
"save_local_shap_values": True,
}
}
assert expected_config == shap_config.get_explainability_config()
def shap_config():
return SHAPConfig(
baseline=SHAP_BASELINE,
num_samples=SHAP_NUM_OF_SAMPLES,
agg_method=SHAP_AGG_METHOD,
save_local_shap_values=False,
)
def test_invalid_shap_config():
with pytest.raises(ValueError) as error:
SHAPConfig(
baseline=[[1]],
num_samples=1,
agg_method="invalid",
)
assert "Invalid agg_method invalid. Please choose mean_abs, median, or mean_sq." in str(
error.value
)
with pytest.raises(ValueError) as error:
SHAPConfig(baseline=[[1]], num_samples=1, agg_method="mean_abs", num_clusters=2)
assert (
"Baseline and num_clusters cannot be provided together. Please specify one of the two."
in str(error.value)
)
def shap_config():
return SHAPConfig(
baseline=[],
num_samples=15,
agg_method="mean_abs",
save_local_shap_values=True,
)
def test_invalid_shap_config():
with pytest.raises(ValueError) as error:
SHAPConfig(
baseline=[[1]],
num_samples=1,
agg_method="invalid",
)
assert "Invalid agg_method invalid. Please choose mean_abs, median, or mean_sq." in str(
error.value)
def shap_config():
return SHAPConfig(
baseline=[[
0.26124998927116394,
0.2824999988079071,
0.06875000149011612,
]],
num_samples=100,
agg_method="mean_sq",
)
def shap_config():
return SHAPConfig(
baseline=[[
0.94672389,
0.47108862,
0.63350081,
0.00604642,
]],
num_samples=2,
agg_method="mean_sq",
)
def test_shap_config_no_baseline():
num_samples = 100
agg_method = "mean_sq"
use_logit = True
seed = 123
shap_config = SHAPConfig(
num_samples=num_samples,
agg_method=agg_method,
num_clusters=2,
use_logit=use_logit,
seed=seed,
)
expected_config = {
"shap": {
"num_samples": num_samples,
"agg_method": agg_method,
"num_clusters": 2,
"use_logit": use_logit,
"save_local_shap_values": True,
"seed": seed,
}
}
assert expected_config == shap_config.get_explainability_config()
def test_shap_with_text_config(
name_from_base,
clarify_processor,
clarify_processor_with_job_name_prefix,
data_config,
model_config,
):
granularity = "paragraph"
language = "ukrainian"
shap_config = SHAPConfig(
baseline=[
[
0.26124998927116394,
0.2824999988079071,
0.06875000149011612,
]
],
num_samples=100,
agg_method="mean_sq",
text_config=TextConfig(granularity, language),
)
expected_text_config = {
"granularity": granularity,
"language": language,
}
expected_predictor_config = {
"model_name": "xgboost-model",
"instance_type": "ml.c5.xlarge",
"initial_instance_count": 1,
}
_run_test_explain(
name_from_base,
clarify_processor,
clarify_processor_with_job_name_prefix,
data_config,
model_config,
shap_config,
None,
None,
expected_predictor_config,
expected_text_config=expected_text_config,
)
def test_shap_config():
baseline = [[
0.26124998927116394,
0.2824999988079071,
0.06875000149011612,
]]
num_samples = 100
agg_method = "mean_sq"
use_logit = True
seed = 123
granularity = "sentence"
language = "german"
model_type = "IMAGE_CLASSIFICATION"
num_segments = 2
feature_extraction_method = "segmentation"
segment_compactness = 10
max_objects = 4
iou_threshold = 0.5
context = 1.0
text_config = TextConfig(
granularity=granularity,
language=language,
)
image_config = ImageConfig(
model_type=model_type,
num_segments=num_segments,
feature_extraction_method=feature_extraction_method,
segment_compactness=segment_compactness,
max_objects=max_objects,
iou_threshold=iou_threshold,
context=context,
)
shap_config = SHAPConfig(
baseline=baseline,
num_samples=num_samples,
agg_method=agg_method,
use_logit=use_logit,
seed=seed,
text_config=text_config,
image_config=image_config,
)
expected_config = {
"shap": {
"baseline": baseline,
"num_samples": num_samples,
"agg_method": agg_method,
"use_logit": use_logit,
"save_local_shap_values": True,
"seed": seed,
"text_config": {
"granularity": granularity,
"language": language,
},
"image_config": {
"model_type": model_type,
"num_segments": num_segments,
"feature_extraction_method": feature_extraction_method,
"segment_compactness": segment_compactness,
"max_objects": max_objects,
"iou_threshold": iou_threshold,
"context": context,
},
}
}
assert expected_config == shap_config.get_explainability_config()
def test_shap_with_image_config(
name_from_base,
clarify_processor,
clarify_processor_with_job_name_prefix,
data_config,
model_config,
):
model_type = "IMAGE_CLASSIFICATION"
num_segments = 2
feature_extraction_method = "segmentation"
segment_compactness = 10
max_objects = 4
iou_threshold = 0.5
context = 1.0
image_config = ImageConfig(
model_type=model_type,
num_segments=num_segments,
feature_extraction_method=feature_extraction_method,
segment_compactness=segment_compactness,
max_objects=max_objects,
iou_threshold=iou_threshold,
context=context,
)
shap_config = SHAPConfig(
baseline=[[
0.26124998927116394,
0.2824999988079071,
0.06875000149011612,
]],
num_samples=100,
agg_method="mean_sq",
image_config=image_config,
)
expected_image_config = {
"model_type": model_type,
"num_segments": num_segments,
"feature_extraction_method": feature_extraction_method,
"segment_compactness": segment_compactness,
"max_objects": max_objects,
"iou_threshold": iou_threshold,
"context": context,
}
expected_predictor_config = {
"model_name": "xgboost-model",
"instance_type": "ml.c5.xlarge",
"initial_instance_count": 1,
}
_run_test_explain(
name_from_base,
clarify_processor,
clarify_processor_with_job_name_prefix,
data_config,
model_config,
shap_config,
None,
None,
expected_predictor_config,
expected_image_config=expected_image_config,
)
