def test_simple_lime(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[20.0, 50.0, 30.0]], requires_grad=True) self._lime_test_assert( net, inp, [73.3716, 193.3349, 113.3349], perturbations_per_eval=(1, 2, 3), n_perturb_samples=500, expected_coefs_only=[73.3716, 193.3349, 113.3349], test_generator=True, )
def test_lrp_multi_inputs(self) -> None: model = BasicModel_MultiLayer() input = torch.Tensor([[1, 2, 3]]) input = (input, 3 * input) lrp = LRP(model) attributions, delta = lrp.attribute(input, target=0, return_convergence_delta=True) self.assertEqual(len(input), 2) assertTensorAlmostEqual(self, attributions[0], torch.Tensor([[16, 32, 48]])) assertTensorAlmostEqual(self, delta, torch.Tensor([-104.0]))
def test_simple_batch_kernel_shap_with_mask(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[2.0, 10.0, 3.0], [20.0, 50.0, 30.0]], requires_grad=True) self._kernel_shap_test_assert( net, inp, [[39.5, 39.5, 10.5], [275.0, 275.0, 115.0]], feature_mask=torch.tensor([[0, 0, 1], [1, 1, 0]]), perturbations_per_eval=(1, 2, 3), n_perturb_samples=100, expected_coefs=[[39.5, 10.5], [115.0, 275.0]], )
def test_multi_sample_ablation_with_mask(self) -> None: ablation_algo = FeatureAblation(BasicModel_MultiLayer()) inp = torch.tensor([[2.0, 10.0, 3.0], [20.0, 50.0, 30.0]], requires_grad=True) mask = torch.tensor([[0, 0, 1], [1, 1, 0]]) self._ablation_test_assert( ablation_algo, inp, [[41.0, 41.0, 12.0], [280.0, 280.0, 120.0]], feature_mask=mask, perturbations_per_eval=(1, 2, 3), )
def test_multi_sample_shapley_sampling_with_mask(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[2.0, 10.0, 3.0], [20.0, 50.0, 30.0]], requires_grad=True) mask = torch.tensor([[0, 0, 1], [1, 1, 0]]) self._shapley_test_assert( net, inp, [[39.5, 39.5, 10.5], [275.0, 275.0, 115.0]], feature_mask=mask, perturbations_per_eval=(1, 2, 3), )
def test_simple_lime_with_baselines(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[20.0, 50.0, 30.0]]) self._lime_test_assert( net, inp, [244.0, 244.0, 100.0], feature_mask=torch.tensor([[0, 0, 1]]), baselines=4, perturbations_per_eval=(1, 2, 3), expected_coefs_only=[244.0, 100.0], )
def test_layer_gradient_linear0(self) -> None: model = BasicModel_MultiLayer() input = torch.tensor([[5.0, -11.0, 23.0]], requires_grad=True) grads, eval = compute_layer_gradients_and_eval(model, model.linear0, input, target_ind=0) assertArraysAlmostEqual(grads[0].squeeze(0).tolist(), [4.0, 4.0, 4.0], delta=0.01) assertArraysAlmostEqual(eval[0].squeeze(0).tolist(), [5.0, -11.0, 23.0], delta=0.01)
def test_simple_lime_with_mask(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[20.0, 50.0, 30.0]], requires_grad=True) self._lime_test_assert( net, inp, [271.0, 271.0, 111.0], feature_mask=torch.tensor([[0, 0, 1]]), perturbations_per_eval=(1, 2, 3), n_perturb_samples=500, expected_coefs_only=[271.0, 111.0], )
def test_attack_comparator_with_preproc(self) -> None: model = BasicModel_MultiLayer() text_inp = ["abc", "zyd", "ghi"] attack_comp = AttackComparator(forward_func=model, metric=tuple_metric, preproc_fn=text_preproc_fn) attack_comp.add_attack( SamplePerturb().perturb, name="Sequence Column Perturb", num_attempts=5, apply_before_preproc=False, ) attack_comp.add_attack( string_perturb, name="StringPerturb", apply_before_preproc=True, ) batch_results = attack_comp.evaluate(text_inp, target=0, named_tuple=True, perturbations_per_eval=3) expected_first_results = { "Original": (0.0, 1280.0), "Sequence Column Perturb": { "mean": (0.0, 847.2), "max": (0.0, 892.0), "min": (0.0, 792.0), }, "StringPerturb": { "mean": (0.0, 1156.0) }, } self._compare_results(batch_results, expected_first_results) expected_summary_results = { "Original": { "mean": (0.0, 1280.0) }, "Sequence Column Perturb Mean Attempt": { "mean": (0.0, 847.2) }, "Sequence Column Perturb Min Attempt": { "mean": (0.0, 792.0) }, "Sequence Column Perturb Max Attempt": { "mean": (0.0, 892.0) }, "StringPerturb": { "mean": (0.0, 1156.0) }, } self._compare_results(attack_comp.summary(), expected_summary_results)
def test_classification_infidelity_tpl_target_w_baseline(self) -> None: model = BasicModel_MultiLayer() input = torch.arange(1.0, 13.0).view(4, 3) baseline = torch.ones(4, 3) additional_forward_args = (torch.arange(1, 13).view(4, 3).float(), True) targets: List = [(0, 1, 1), (0, 1, 1), (1, 1, 1), (0, 1, 1)] ig = IntegratedGradients(model) def perturbed_func2(inputs, baselines): return torch.ones(baselines.shape), baselines @infidelity_perturb_func_decorator(True) def perturbed_func3(inputs, baselines): return baselines attr, delta = ig.attribute( input, target=targets, additional_forward_args=additional_forward_args, baselines=baseline, return_convergence_delta=True, ) infid = self.infidelity_assert( model, attr, input, torch.tensor([0.10686, 0.0, 0.0, 0.0]), additional_args=additional_forward_args, baselines=baseline, target=targets, multi_input=False, n_perturb_samples=3, perturb_func=perturbed_func3, ) infid2 = self.infidelity_assert( model, attr, input, torch.tensor([0.10686, 0.0, 0.0, 0.0]), additional_args=additional_forward_args, baselines=baseline, target=targets, multi_input=False, n_perturb_samples=3, perturb_func=perturbed_func2, ) assertTensorAlmostEqual(self, infid, delta * delta) assertTensorAlmostEqual(self, infid, infid2)
def test_linear_kernel_shap(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[20.0, 50.0, 30.0]], requires_grad=True) baseline = torch.tensor([[10.0, 20.0, 10.0]], requires_grad=True) self._kernel_shap_test_assert( net, inp, [[40.0, 120.0, 80.0]], n_samples=500, baselines=baseline, expected_coefs=[[40.0, 120.0, 80.0]], )
def test_error_agg_mode_arbitrary_output(self) -> None: net = BasicModel_MultiLayer() # output 3 numbers for the entire batch # note that the batch size == 2 def forward_func(inp): pred = net(inp) return torch.stack([pred.sum(), pred.max(), pred.min()]) inp = torch.tensor([[2.0, 10.0, 3.0], [20.0, 50.0, 30.0]], requires_grad=True) ablation = FeatureAblation(forward_func) with self.assertRaises(AssertionError): _ = ablation.attribute(inp, perturbations_per_eval=2)
def test_simple_batch_lime(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[20.0, 50.0, 30.0], [10.0, 14.0, 4.0]], requires_grad=True) self._lime_test_assert( net, inp, [[73.4450, 193.5979, 113.4363], [32.11, 48.00, 11.00]], perturbations_per_eval=(1, 2, 3), n_perturb_samples=800, expected_coefs_only=[[73.4450, 193.5979, 113.4363], [32.11, 48.00, 11.00]], )
def test_minimal_pert_additional_forward_args(self) -> None: model = BasicModel_MultiLayer() text_inp = [["abc", "zyd", "ghi"], ["abc", "uyd", "ghi"]] additional_forward_args = torch.ones((2, 3)) * -97 model = BasicModel_MultiLayer() minimal_pert = MinParamPerturbation( forward_func=model, attack=add_char_batch, arg_name="char_val", arg_min=0, arg_max=26, arg_step=1, preproc_fn=batch_text_preproc_fn, apply_before_preproc=True, correct_fn=alt_correct_fn, ) expected_list = [["abc", "uzyd", "ghi"], ["abc", "uuyd", "ghi"]] target_inp, pert = minimal_pert.evaluate( text_inp, target=1, attack_kwargs={"ind": 1}, correct_fn_kwargs={"threshold": 100}, perturbations_per_eval=15, additional_forward_args=(additional_forward_args,), ) self.assertEqual(pert, 5) self.assertListEqual(target_inp, expected_list) target_inp_single, pert_single = minimal_pert.evaluate( text_inp, target=1, attack_kwargs={"ind": 1}, correct_fn_kwargs={"threshold": 100}, additional_forward_args=(additional_forward_args,), ) self.assertEqual(pert_single, 5) self.assertListEqual(target_inp_single, expected_list)
def test_simple_multi_layer_multi_output_activation(self) -> None: net = BasicModel_MultiLayer(multi_input_module=True) inp = torch.tensor([[0.0, 6.0, 0.0]]) self._multiple_layer_activation_test_assert( net, [net.multi_relu, net.linear0, net.linear1], inp, [ ([0.0, 7.0, 7.0, 7.0], [0.0, 7.0, 7.0, 7.0]), [[0.0, 6.0, 0.0]], [[-4.0, 7.0, 7.0, 7.0]], ], )
def test_simple_batch_lime_with_mask(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[20.0, 50.0, 30.0], [10.0, 14.0, 4.0]], requires_grad=True) self._lime_test_assert( net, inp, [[271.0, 271.0, 111.0], [32.11, 48.00, 11.00]], feature_mask=torch.tensor([[0, 0, 1], [0, 1, 2]]), perturbations_per_eval=(1, 2, 3), n_samples=600, expected_coefs_only=[[271.0, 111.0, 0.0], [32.11, 48.00, 11.00]], test_generator=True, )
def test_basic_multilayer_compare_w_inp_features(self) -> None: model = BasicModel_MultiLayer() model.eval() inputs = torch.tensor([[10.0, 20.0, 10.0]]) baselines = torch.randn(30, 3) gs = GradientShap(model) expected, delta = gs.attribute(inputs, baselines, target=0, return_convergence_delta=True) self.setUp() self._assert_attributions( model, model.linear0, inputs, baselines, 0, (expected, ), expected_delta=delta, attribute_to_layer_input=True, )
def test_multi_sample_ablation_with_slice(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[2.0, 10.0, 3.0], [20.0, 50.0, 30.0]], requires_grad=True) mask = torch.tensor([[0, 0, 1], [1, 1, 0]]) self._ablation_test_assert( net, net.linear2, inp, [[82.0, 82.0, 24.0], [560.0, 560.0, 240.0]], feature_mask=mask, perturbations_per_eval=(1, 2, 3), neuron_selector=(slice(0, 2, 1), ), )
def test_lrp_multi(self) -> None: model = BasicModel_MultiLayer() input = torch.Tensor([[1, 2, 3]]) add_input = 0 output = model(input) output_add = model(input, add_input=add_input) self.assertTrue(torch.equal(output, output_add)) lrp = LRP(model) attributions = lrp.attribute(input, target=0) attributions_add_input = lrp.attribute( input, target=0, additional_forward_args=(add_input, )) self.assertTrue( torch.equal(attributions, attributions_add_input) # type: ignore ) # type: ignore
def test_attack_additional_inputs(self) -> None: model = BasicModel_MultiLayer() add_input = torch.tensor([[-1.0, 2.0, 2.0]], requires_grad=True) input = torch.tensor([[1.0, 6.0, -3.0]], requires_grad=True) self._FGSM_assert(model, input, 0, 0.2, [[0.8, 5.8, -3.2]], additional_inputs=(add_input, )) self._FGSM_assert(model, input, 0, 0.2, [[0.8, 5.8, -3.2]], additional_inputs=add_input)
def test_attack_loss_defined(self) -> None: model = BasicModel_MultiLayer() add_input = torch.tensor([[-1.0, 2.0, 2.0]]) input = torch.tensor([[1.0, 6.0, -3.0]]) labels = torch.tensor([0]) loss_func = CrossEntropyLoss(reduction="none") adv = FGSM(model, loss_func) perturbed_input = adv.perturb(input, 0.2, labels, additional_forward_args=(add_input, )) assertArraysAlmostEqual(perturbed_input.squeeze(0).tolist(), [1.0, 6.0, -3.0], delta=0.01)
def test_layer_gradient_relu_input_inplace(self) -> None: model = BasicModel_MultiLayer(inplace=True) input = torch.tensor([[5.0, 2.0, 1.0]], requires_grad=True) grads, eval = compute_layer_gradients_and_eval( model, model.relu, input, target_ind=1, attribute_to_layer_input=True) assertArraysAlmostEqual(grads[0].squeeze(0).tolist(), [0.0, 1.0, 1.0, 1.0], delta=0.01) assertArraysAlmostEqual(eval[0].squeeze(0).tolist(), [-2.0, 9.0, 9.0, 9.0], delta=0.01)
def _assert_batched_tensor_input( self, type: str, approximation_method: str = "gausslegendre" ) -> None: model = BasicModel_MultiLayer() input = ( torch.tensor( [[1.5, 2.0, 1.3], [0.5, 0.1, 2.3], [1.5, 2.0, 1.3]], requires_grad=True ), ) self._compute_attribution_and_evaluate( model, input, type=type, target=0, approximation_method=approximation_method ) self._compute_attribution_batch_helper_evaluate( model, input, target=0, approximation_method=approximation_method )
def test_relu_layer_deeplift_multiple_output(self) -> None: model = BasicModel_MultiLayer(multi_input_module=True) inputs, baselines = _create_inps_and_base_for_deeplift_neuron_layer_testing( ) layer_dl = LayerDeepLift(model, model.multi_relu) attributions, delta = layer_dl.attribute( inputs[0], baselines[0], target=0, attribute_to_layer_input=False, return_convergence_delta=True, ) assertTensorTuplesAlmostEqual( self, attributions, ([[0.0, -1.0, -1.0, -1.0]], [[0.0, -1.0, -1.0, -1.0]])) assert_delta(self, delta)
def test_attack_loss_defined(self) -> None: model = BasicModel_MultiLayer() add_input = torch.tensor([[-1.0, 2.0, 2.0]]) input = torch.tensor([[1.0, 6.0, -3.0]]) labels = torch.tensor([0]) loss_func = CrossEntropyLoss(reduction="none") adv = PGD(model, loss_func) perturbed_input = adv.perturb(input, 0.25, 0.1, 3, labels, additional_forward_args=(add_input, )) assertTensorAlmostEqual(self, perturbed_input, [[1.0, 6.0, -3.0]], delta=0.01, mode="max")
def test_minimal_pert_preproc(self) -> None: model = BasicModel_MultiLayer() text_inp = ["abc", "zyd", "ghi"] minimal_pert = MinParamPerturbation( forward_func=model, attack=add_char, arg_name="char_val", arg_min=0, arg_max=26, arg_step=1, preproc_fn=text_preproc_fn, apply_before_preproc=True, ) target_inp, pert = minimal_pert.evaluate( text_inp, target=1, attack_kwargs={"ind": 1} ) self.assertEqual(pert, None) self.assertEqual(target_inp, None)
def _assert_n_samples_batched_size( self, type: str, approximation_method: str = "gausslegendre", nt_samples_batch_size: int = None, ) -> None: model = BasicModel_MultiLayer() input = (torch.tensor( [[1.5, 2.0, 1.3], [0.5, 0.1, 2.3], [1.5, 2.0, 1.3]], requires_grad=True), ) self._compute_attribution_and_evaluate( model, input, type=type, target=0, nt_samples_batch_size=nt_samples_batch_size, approximation_method=approximation_method, )
def test_sensitivity_max_multi_dim(self) -> None: model = BasicModel_MultiLayer() input = torch.arange(1.0, 13.0).view(4, 3) additional_forward_args = (None, True) targets: List = [(0, 1, 1), (0, 1, 1), (1, 1, 1), (0, 1, 1)] ig = IntegratedGradients(model) self.sensitivity_max_assert( ig.attribute, input, torch.tensor([0.006, 0.01, 0.001, 0.008]), n_perturb_samples=1, max_examples_per_batch=4, perturb_func=_perturb_func, target=targets, additional_forward_args=additional_forward_args, )
def _assert_batched_tensor_multi_input( self, type: str, approximation_method: str = "gausslegendre", nt_samples_batch_size: int = None, ) -> None: model = BasicModel_MultiLayer() input = ( torch.tensor([[1.5, 2.1, 1.9], [0.5, 0.0, 0.7], [1.5, 2.1, 1.1]], requires_grad=True), torch.tensor([[0.3, 1.9, 2.4], [0.5, 0.6, 2.1], [1.2, 2.1, 0.2]], requires_grad=True), ) self._compute_attribution_and_evaluate( model, input, type=type, target=0, approximation_method=approximation_method, nt_samples_batch_size=nt_samples_batch_size, )
def test_multiple_linear_internal_inf(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor( [ [0.0, 100.0, 0.0], [0.0, 100.0, 0.0], [0.0, 100.0, 0.0], [0.0, 100.0, 0.0], ], requires_grad=True, ) self._internal_influence_test_assert( net, net.linear1, inp, [ [0.9, 1.0, 1.0, 1.0], [0.9, 1.0, 1.0, 1.0], [0.9, 1.0, 1.0, 1.0], [0.9, 1.0, 1.0, 1.0], ], )