def test_convnet_maxpool2d_classification(self) -> None:
inputs = 100 * torch.randn(2, 1, 10, 10)
model = BasicModel_ConvNet()
model.eval()
dl = LayerDeepLift(model, model.pool1)
dl2 = LayerDeepLift(model, model.conv2)
attr = dl.attribute(inputs, target=0)
attr2 = dl2.attribute(inputs, target=0, attribute_to_layer_input=True)
self.assertTrue(cast(Tensor, attr).sum() == cast(Tensor, attr2).sum())
def test_matching_conv2_multi_input_conductance(self) -> None:
net = BasicModel_ConvNet()
inp = 100 * torch.randn(2, 1, 10, 10)
self._conductance_input_sum_test_assert(net, net.conv2, inp, 0.0)
# trying different baseline
self._conductance_input_sum_test_assert(net, net.conv2, inp, 0.000001)
def test_convnet_with_maxpool2d_large_baselines(self) -> None:
input = 100 * torch.randn(2, 1, 10, 10, requires_grad=True)
baseline = 500 * torch.randn(2, 1, 10, 10)
model = BasicModel_ConvNet()
dl = DeepLift(model)
self.softmax_classification(model, dl, input, baseline,
torch.tensor(2))
def create_TCAV(save_path: str, classifier: Classifier, layers) -> TCAV:
model = BasicModel_ConvNet()
tcav = TCAV(
model,
layers,
classifier,
save_path=save_path,
)
return tcav
def test_convnet_maxpool2d_classification(self) -> None:
inputs = 100 * torch.randn(2, 1, 10, 10)
model = BasicModel_ConvNet()
ndl = NeuronDeepLift(model, model.pool1)
attr = ndl.attribute(inputs, neuron_selector=(0, 0, 0))
ndl2 = NeuronDeepLift(model, model.conv2)
attr2 = ndl2.attribute(inputs,
neuron_selector=(0, 0, 0),
attribute_to_neuron_input=True)
assertTensorAlmostEqual(self, attr.sum(), attr2.sum())
def create_TCAV(
save_path: str,
classifier: Classifier,
layers: Union[str, List[str]],
attribute_to_layer_input: bool = False,
) -> TCAV:
model = BasicModel_ConvNet()
tcav = TCAV(
model,
layers,
classifier=classifier,
save_path=save_path,
attribute_to_layer_input=attribute_to_layer_input,
)
return tcav
def test_matching_intermediate_gradient(self) -> None:
net = BasicModel_ConvNet()
inp = torch.randn(3, 1, 10, 10)
self._gradient_matching_test_assert(net, net.relu2, inp)
def test_matching_output_gradient(self) -> None:
net = BasicModel_ConvNet()
inp = torch.randn(2, 1, 10, 10, requires_grad=True)
self._gradient_matching_test_assert(net, net.softmax, inp)
"stdevs": 0.0,
},
"target_delta": 0.6,
"baseline_distr": True,
},
# Perturbation-Specific Configs
{
"name": "conv_with_perturbations_per_eval",
"algorithms": [
FeatureAblation,
ShapleyValueSampling,
FeaturePermutation,
Lime,
KernelShap,
],
"model": BasicModel_ConvNet(),
"attribute_args": {
"inputs": torch.arange(400).view(4, 1, 10, 10).float(),
"target": 0,
"perturbations_per_eval": 20,
},
"dp_delta": 0.008,
},
{
"name": "basic_multiple_tuple_target_with_perturbations_per_eval",
"algorithms": [
FeatureAblation,
ShapleyValueSampling,
FeaturePermutation,
Lime,
KernelShap,
def test_matching_pool2_multi_input_conductance(self) -> None:
net = BasicModel_ConvNet()
inp = 100 * torch.randn(1, 1, 10, 10)
baseline = 20 * torch.randn(1, 1, 10, 10, requires_grad=True)
self._conductance_input_sum_test_assert(net, net.pool2, inp, baseline)
def test_matching_relu2_with_scalar_base_multi_input_conductance(
self) -> None:
net = BasicModel_ConvNet()
inp = 100 * torch.randn(3, 1, 10, 10, requires_grad=True)
self._conductance_input_sum_test_assert(net, net.relu2, inp, 0.0)
def test_matching_conv_with_baseline_conductance(self) -> None:
net = BasicModel_ConvNet()
inp = 100 * torch.randn(3, 1, 10, 10)
baseline = 100 * torch.randn(3, 1, 10, 10, requires_grad=True)
self._conductance_reference_test_assert(net, net.fc1, inp, baseline)
def test_matching_conv_multi_input_conductance(self) -> None:
net = BasicModel_ConvNet()
inp = 100 * torch.randn(4, 1, 10, 10, requires_grad=True)
self._conductance_reference_test_assert(net, net.relu3, inp)
def test_matching_pool2_conductance(self) -> None:
net = BasicModel_ConvNet()
inp = 100 * torch.randn(1, 1, 10, 10)
self._conductance_reference_test_assert(net, net.pool2, inp)
def test_exp_sets_with_diffent_lengths(self) -> None:
# Create Concepts
concepts, concepts_dict = create_concepts()
# defining experimental sets of different length
experimental_set_list = [["striped", "random"], ["ceo", "striped", "random"]]
experimental_sets_diff_length = self._create_experimental_sets(
experimental_set_list, concepts_dict
)
exp_sets_striped_random = self._create_experimental_sets(
[["striped", "random"]], concepts_dict
)
exp_sets_ceo_striped_random = self._create_experimental_sets(
[["ceo", "striped", "random"]], concepts_dict
)
striped_random_str = concepts_to_str(exp_sets_striped_random[0])
ceo_striped_random_str = concepts_to_str(exp_sets_ceo_striped_random[0])
model = BasicModel_ConvNet()
model.eval()
layers = ["conv1", "conv2", "fc1", "fc2"]
inputs = torch.randn(5, 1, 10, 10)
with tempfile.TemporaryDirectory() as tmpdirname:
tcav_diff_length = TCAV(
model,
layers,
save_path=tmpdirname,
)
# computing tcav scores for `striped and random` set and
# `ceo, striped and random` set at once using one `interpret`
# call.
interpret_diff_lengths = tcav_diff_length.interpret(
inputs, experimental_sets=experimental_sets_diff_length, target=0
)
# computing tcav scores for striped and random
interpret_striped_random = tcav_diff_length.interpret(
inputs, experimental_sets=exp_sets_striped_random, target=0
)
# computing tcav scores for ceo, striped and random
interpret_ceo_striped_random = tcav_diff_length.interpret(
inputs, experimental_sets=exp_sets_ceo_striped_random, target=0
)
for combined, separate in zip(
interpret_diff_lengths[striped_random_str].items(),
interpret_striped_random[striped_random_str].items(),
):
self.assertEqual(combined[0], separate[0])
for c_tcav, s_tcav in zip(combined[1].items(), separate[1].items()):
self.assertEqual(c_tcav[0], s_tcav[0])
assertTensorAlmostEqual(self, c_tcav[1], s_tcav[1])
for combined, separate in zip(
interpret_diff_lengths[ceo_striped_random_str].items(),
interpret_ceo_striped_random[ceo_striped_random_str].items(),
):
self.assertEqual(combined[0], separate[0])
for c_tcav, s_tcav in zip(combined[1].items(), separate[1].items()):
self.assertEqual(c_tcav[0], s_tcav[0])
assertTensorAlmostEqual(self, c_tcav[1], s_tcav[1])
def test_model_ids_in_tcav(self, ) -> None:
# creating concepts and mapping between concepts and their names
concepts_dict = create_concepts()
# defining experimental sets of different length
experimental_set_list = [["striped", "random"], ["dotted", "random"]]
experimental_sets = self._create_experimental_sets(
experimental_set_list, concepts_dict)
model = BasicModel_ConvNet()
model.eval()
layer = "conv2"
inputs = 100 * get_inputs_tensor()
with tempfile.TemporaryDirectory() as tmpdirname:
tcav1 = TCAV(
model,
layer,
model_id="my_basic_model1",
classifier=CustomClassifier(),
save_path=tmpdirname,
)
interpret1 = tcav1.interpret(inputs,
experimental_sets=experimental_sets,
target=0)
tcav2 = TCAV(
model,
layer,
model_id="my_basic_model2",
classifier=CustomClassifier(),
save_path=tmpdirname,
)
interpret2 = tcav2.interpret(inputs,
experimental_sets=experimental_sets,
target=0)
# testing that different folders were created for two different
# ids of the model
self.assertTrue(
AV.exists(
tmpdirname,
"my_basic_model1",
concepts_dict["striped"].identifier,
layer,
))
self.assertTrue(
AV.exists(
tmpdirname,
"my_basic_model2",
concepts_dict["striped"].identifier,
layer,
))
for interpret1_elem, interpret2_elem in zip(
interpret1, interpret2):
for interpret1_sub_elem, interpret2_sub_elem in zip(
interpret1[interpret1_elem],
interpret2[interpret2_elem]):
assertTensorAlmostEqual(
self,
interpret1[interpret1_elem][interpret1_sub_elem]
["sign_count"],
interpret2[interpret2_elem][interpret2_sub_elem]
["sign_count"],
0.0,
)
assertTensorAlmostEqual(
self,
interpret1[interpret1_elem][interpret1_sub_elem]
["magnitude"],
interpret2[interpret2_elem][interpret2_sub_elem]
["magnitude"],
0.0,
)
self.assertEqual(interpret1_sub_elem, interpret2_sub_elem)
self.assertEqual(interpret1_elem, interpret2_elem)
