def test_convnet_maxpool2d_classification(self) -> None: inputs = 100 * torch.randn(2, 1, 10, 10) model = BasicModel_ConvNet() model.eval() dl = LayerDeepLift(model, model.pool1) dl2 = LayerDeepLift(model, model.conv2) attr = dl.attribute(inputs, target=0) attr2 = dl2.attribute(inputs, target=0, attribute_to_layer_input=True) self.assertTrue(cast(Tensor, attr).sum() == cast(Tensor, attr2).sum())
def test_matching_conv2_multi_input_conductance(self) -> None: net = BasicModel_ConvNet() inp = 100 * torch.randn(2, 1, 10, 10) self._conductance_input_sum_test_assert(net, net.conv2, inp, 0.0) # trying different baseline self._conductance_input_sum_test_assert(net, net.conv2, inp, 0.000001)
def test_convnet_with_maxpool2d_large_baselines(self) -> None: input = 100 * torch.randn(2, 1, 10, 10, requires_grad=True) baseline = 500 * torch.randn(2, 1, 10, 10) model = BasicModel_ConvNet() dl = DeepLift(model) self.softmax_classification(model, dl, input, baseline, torch.tensor(2))
def create_TCAV(save_path: str, classifier: Classifier, layers) -> TCAV: model = BasicModel_ConvNet() tcav = TCAV( model, layers, classifier, save_path=save_path, ) return tcav
def test_convnet_maxpool2d_classification(self) -> None: inputs = 100 * torch.randn(2, 1, 10, 10) model = BasicModel_ConvNet() ndl = NeuronDeepLift(model, model.pool1) attr = ndl.attribute(inputs, neuron_selector=(0, 0, 0)) ndl2 = NeuronDeepLift(model, model.conv2) attr2 = ndl2.attribute(inputs, neuron_selector=(0, 0, 0), attribute_to_neuron_input=True) assertTensorAlmostEqual(self, attr.sum(), attr2.sum())
def create_TCAV( save_path: str, classifier: Classifier, layers: Union[str, List[str]], attribute_to_layer_input: bool = False, ) -> TCAV: model = BasicModel_ConvNet() tcav = TCAV( model, layers, classifier=classifier, save_path=save_path, attribute_to_layer_input=attribute_to_layer_input, ) return tcav
def test_matching_intermediate_gradient(self) -> None: net = BasicModel_ConvNet() inp = torch.randn(3, 1, 10, 10) self._gradient_matching_test_assert(net, net.relu2, inp)
def test_matching_output_gradient(self) -> None: net = BasicModel_ConvNet() inp = torch.randn(2, 1, 10, 10, requires_grad=True) self._gradient_matching_test_assert(net, net.softmax, inp)
"stdevs": 0.0, }, "target_delta": 0.6, "baseline_distr": True, }, # Perturbation-Specific Configs { "name": "conv_with_perturbations_per_eval", "algorithms": [ FeatureAblation, ShapleyValueSampling, FeaturePermutation, Lime, KernelShap, ], "model": BasicModel_ConvNet(), "attribute_args": { "inputs": torch.arange(400).view(4, 1, 10, 10).float(), "target": 0, "perturbations_per_eval": 20, }, "dp_delta": 0.008, }, { "name": "basic_multiple_tuple_target_with_perturbations_per_eval", "algorithms": [ FeatureAblation, ShapleyValueSampling, FeaturePermutation, Lime, KernelShap,
def test_matching_pool2_multi_input_conductance(self) -> None: net = BasicModel_ConvNet() inp = 100 * torch.randn(1, 1, 10, 10) baseline = 20 * torch.randn(1, 1, 10, 10, requires_grad=True) self._conductance_input_sum_test_assert(net, net.pool2, inp, baseline)
def test_matching_relu2_with_scalar_base_multi_input_conductance( self) -> None: net = BasicModel_ConvNet() inp = 100 * torch.randn(3, 1, 10, 10, requires_grad=True) self._conductance_input_sum_test_assert(net, net.relu2, inp, 0.0)
def test_matching_conv_with_baseline_conductance(self) -> None: net = BasicModel_ConvNet() inp = 100 * torch.randn(3, 1, 10, 10) baseline = 100 * torch.randn(3, 1, 10, 10, requires_grad=True) self._conductance_reference_test_assert(net, net.fc1, inp, baseline)
def test_matching_conv_multi_input_conductance(self) -> None: net = BasicModel_ConvNet() inp = 100 * torch.randn(4, 1, 10, 10, requires_grad=True) self._conductance_reference_test_assert(net, net.relu3, inp)
def test_matching_pool2_conductance(self) -> None: net = BasicModel_ConvNet() inp = 100 * torch.randn(1, 1, 10, 10) self._conductance_reference_test_assert(net, net.pool2, inp)
def test_exp_sets_with_diffent_lengths(self) -> None: # Create Concepts concepts, concepts_dict = create_concepts() # defining experimental sets of different length experimental_set_list = [["striped", "random"], ["ceo", "striped", "random"]] experimental_sets_diff_length = self._create_experimental_sets( experimental_set_list, concepts_dict ) exp_sets_striped_random = self._create_experimental_sets( [["striped", "random"]], concepts_dict ) exp_sets_ceo_striped_random = self._create_experimental_sets( [["ceo", "striped", "random"]], concepts_dict ) striped_random_str = concepts_to_str(exp_sets_striped_random[0]) ceo_striped_random_str = concepts_to_str(exp_sets_ceo_striped_random[0]) model = BasicModel_ConvNet() model.eval() layers = ["conv1", "conv2", "fc1", "fc2"] inputs = torch.randn(5, 1, 10, 10) with tempfile.TemporaryDirectory() as tmpdirname: tcav_diff_length = TCAV( model, layers, save_path=tmpdirname, ) # computing tcav scores for `striped and random` set and # `ceo, striped and random` set at once using one `interpret` # call. interpret_diff_lengths = tcav_diff_length.interpret( inputs, experimental_sets=experimental_sets_diff_length, target=0 ) # computing tcav scores for striped and random interpret_striped_random = tcav_diff_length.interpret( inputs, experimental_sets=exp_sets_striped_random, target=0 ) # computing tcav scores for ceo, striped and random interpret_ceo_striped_random = tcav_diff_length.interpret( inputs, experimental_sets=exp_sets_ceo_striped_random, target=0 ) for combined, separate in zip( interpret_diff_lengths[striped_random_str].items(), interpret_striped_random[striped_random_str].items(), ): self.assertEqual(combined[0], separate[0]) for c_tcav, s_tcav in zip(combined[1].items(), separate[1].items()): self.assertEqual(c_tcav[0], s_tcav[0]) assertTensorAlmostEqual(self, c_tcav[1], s_tcav[1]) for combined, separate in zip( interpret_diff_lengths[ceo_striped_random_str].items(), interpret_ceo_striped_random[ceo_striped_random_str].items(), ): self.assertEqual(combined[0], separate[0]) for c_tcav, s_tcav in zip(combined[1].items(), separate[1].items()): self.assertEqual(c_tcav[0], s_tcav[0]) assertTensorAlmostEqual(self, c_tcav[1], s_tcav[1])
def test_model_ids_in_tcav(self, ) -> None: # creating concepts and mapping between concepts and their names concepts_dict = create_concepts() # defining experimental sets of different length experimental_set_list = [["striped", "random"], ["dotted", "random"]] experimental_sets = self._create_experimental_sets( experimental_set_list, concepts_dict) model = BasicModel_ConvNet() model.eval() layer = "conv2" inputs = 100 * get_inputs_tensor() with tempfile.TemporaryDirectory() as tmpdirname: tcav1 = TCAV( model, layer, model_id="my_basic_model1", classifier=CustomClassifier(), save_path=tmpdirname, ) interpret1 = tcav1.interpret(inputs, experimental_sets=experimental_sets, target=0) tcav2 = TCAV( model, layer, model_id="my_basic_model2", classifier=CustomClassifier(), save_path=tmpdirname, ) interpret2 = tcav2.interpret(inputs, experimental_sets=experimental_sets, target=0) # testing that different folders were created for two different # ids of the model self.assertTrue( AV.exists( tmpdirname, "my_basic_model1", concepts_dict["striped"].identifier, layer, )) self.assertTrue( AV.exists( tmpdirname, "my_basic_model2", concepts_dict["striped"].identifier, layer, )) for interpret1_elem, interpret2_elem in zip( interpret1, interpret2): for interpret1_sub_elem, interpret2_sub_elem in zip( interpret1[interpret1_elem], interpret2[interpret2_elem]): assertTensorAlmostEqual( self, interpret1[interpret1_elem][interpret1_sub_elem] ["sign_count"], interpret2[interpret2_elem][interpret2_sub_elem] ["sign_count"], 0.0, ) assertTensorAlmostEqual( self, interpret1[interpret1_elem][interpret1_sub_elem] ["magnitude"], interpret2[interpret2_elem][interpret2_sub_elem] ["magnitude"], 0.0, ) self.assertEqual(interpret1_sub_elem, interpret2_sub_elem) self.assertEqual(interpret1_elem, interpret2_elem)