def test_shap_config(): baseline = [[ 0.26124998927116394, 0.2824999988079071, 0.06875000149011612, ]] num_samples = 100 agg_method = "mean_sq" use_logit = True seed = 123 shap_config = SHAPConfig( baseline=baseline, num_samples=num_samples, agg_method=agg_method, use_logit=use_logit, seed=seed, ) expected_config = { "shap": { "baseline": baseline, "num_samples": num_samples, "agg_method": agg_method, "use_logit": use_logit, "save_local_shap_values": True, "seed": seed, } } assert expected_config == shap_config.get_explainability_config()
def test_shap_config_no_parameters(): shap_config = SHAPConfig() expected_config = { "shap": { "use_logit": False, "save_local_shap_values": True, } } assert expected_config == shap_config.get_explainability_config()
def shap_config(): return SHAPConfig( baseline=SHAP_BASELINE, num_samples=SHAP_NUM_OF_SAMPLES, agg_method=SHAP_AGG_METHOD, save_local_shap_values=False, )
def test_invalid_shap_config(): with pytest.raises(ValueError) as error: SHAPConfig( baseline=[[1]], num_samples=1, agg_method="invalid", ) assert "Invalid agg_method invalid. Please choose mean_abs, median, or mean_sq." in str( error.value ) with pytest.raises(ValueError) as error: SHAPConfig(baseline=[[1]], num_samples=1, agg_method="mean_abs", num_clusters=2) assert ( "Baseline and num_clusters cannot be provided together. Please specify one of the two." in str(error.value) )
def shap_config(): return SHAPConfig( baseline=[], num_samples=15, agg_method="mean_abs", save_local_shap_values=True, )
def test_invalid_shap_config(): with pytest.raises(ValueError) as error: SHAPConfig( baseline=[[1]], num_samples=1, agg_method="invalid", ) assert "Invalid agg_method invalid. Please choose mean_abs, median, or mean_sq." in str( error.value)
def shap_config(): return SHAPConfig( baseline=[[ 0.26124998927116394, 0.2824999988079071, 0.06875000149011612, ]], num_samples=100, agg_method="mean_sq", )
def shap_config(): return SHAPConfig( baseline=[[ 0.94672389, 0.47108862, 0.63350081, 0.00604642, ]], num_samples=2, agg_method="mean_sq", )
def test_shap_config_no_baseline(): num_samples = 100 agg_method = "mean_sq" use_logit = True seed = 123 shap_config = SHAPConfig( num_samples=num_samples, agg_method=agg_method, num_clusters=2, use_logit=use_logit, seed=seed, ) expected_config = { "shap": { "num_samples": num_samples, "agg_method": agg_method, "num_clusters": 2, "use_logit": use_logit, "save_local_shap_values": True, "seed": seed, } } assert expected_config == shap_config.get_explainability_config()
def test_shap_with_text_config( name_from_base, clarify_processor, clarify_processor_with_job_name_prefix, data_config, model_config, ): granularity = "paragraph" language = "ukrainian" shap_config = SHAPConfig( baseline=[ [ 0.26124998927116394, 0.2824999988079071, 0.06875000149011612, ] ], num_samples=100, agg_method="mean_sq", text_config=TextConfig(granularity, language), ) expected_text_config = { "granularity": granularity, "language": language, } expected_predictor_config = { "model_name": "xgboost-model", "instance_type": "ml.c5.xlarge", "initial_instance_count": 1, } _run_test_explain( name_from_base, clarify_processor, clarify_processor_with_job_name_prefix, data_config, model_config, shap_config, None, None, expected_predictor_config, expected_text_config=expected_text_config, )
def test_shap_config(): baseline = [[ 0.26124998927116394, 0.2824999988079071, 0.06875000149011612, ]] num_samples = 100 agg_method = "mean_sq" use_logit = True seed = 123 granularity = "sentence" language = "german" model_type = "IMAGE_CLASSIFICATION" num_segments = 2 feature_extraction_method = "segmentation" segment_compactness = 10 max_objects = 4 iou_threshold = 0.5 context = 1.0 text_config = TextConfig( granularity=granularity, language=language, ) image_config = ImageConfig( model_type=model_type, num_segments=num_segments, feature_extraction_method=feature_extraction_method, segment_compactness=segment_compactness, max_objects=max_objects, iou_threshold=iou_threshold, context=context, ) shap_config = SHAPConfig( baseline=baseline, num_samples=num_samples, agg_method=agg_method, use_logit=use_logit, seed=seed, text_config=text_config, image_config=image_config, ) expected_config = { "shap": { "baseline": baseline, "num_samples": num_samples, "agg_method": agg_method, "use_logit": use_logit, "save_local_shap_values": True, "seed": seed, "text_config": { "granularity": granularity, "language": language, }, "image_config": { "model_type": model_type, "num_segments": num_segments, "feature_extraction_method": feature_extraction_method, "segment_compactness": segment_compactness, "max_objects": max_objects, "iou_threshold": iou_threshold, "context": context, }, } } assert expected_config == shap_config.get_explainability_config()
def test_shap_with_image_config( name_from_base, clarify_processor, clarify_processor_with_job_name_prefix, data_config, model_config, ): model_type = "IMAGE_CLASSIFICATION" num_segments = 2 feature_extraction_method = "segmentation" segment_compactness = 10 max_objects = 4 iou_threshold = 0.5 context = 1.0 image_config = ImageConfig( model_type=model_type, num_segments=num_segments, feature_extraction_method=feature_extraction_method, segment_compactness=segment_compactness, max_objects=max_objects, iou_threshold=iou_threshold, context=context, ) shap_config = SHAPConfig( baseline=[[ 0.26124998927116394, 0.2824999988079071, 0.06875000149011612, ]], num_samples=100, agg_method="mean_sq", image_config=image_config, ) expected_image_config = { "model_type": model_type, "num_segments": num_segments, "feature_extraction_method": feature_extraction_method, "segment_compactness": segment_compactness, "max_objects": max_objects, "iou_threshold": iou_threshold, "context": context, } expected_predictor_config = { "model_name": "xgboost-model", "instance_type": "ml.c5.xlarge", "initial_instance_count": 1, } _run_test_explain( name_from_base, clarify_processor, clarify_processor_with_job_name_prefix, data_config, model_config, shap_config, None, None, expected_predictor_config, expected_image_config=expected_image_config, )