def test_exists_without_version(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
av_0 = torch.randn(64, 16)
self.assertFalse(AV.exists(tmpdir, "dummy", "layer1.0.conv1"))
AV.save(tmpdir, "dummy", DEFAULT_IDENTIFIER, "layer1.0.conv1",
av_0, "0")
self.assertTrue(
AV.exists(
tmpdir,
"dummy",
DEFAULT_IDENTIFIER,
"layer1.0.conv1",
))
def test_exists_with_version(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
idf1 = str(int(datetime.now().microsecond))
idf2 = "idf2"
av_0 = torch.randn(64, 16)
self.assertFalse(AV.exists(tmpdir, "dummy", "layer1.0.conv1",
idf1))
self.assertFalse(AV.exists(tmpdir, "dummy", "layer1.0.conv1",
idf2))
AV.save(tmpdir, "dummy", idf1, "layer1.0.conv1", av_0, "0")
self.assertTrue(AV.exists(tmpdir, "dummy", idf1, "layer1.0.conv1"))
self.assertFalse(AV.exists(tmpdir, "dummy", idf2,
"layer1.0.conv1"))
AV.save(tmpdir, "dummy", idf2, "layer1.0.conv1", av_0, "0")
self.assertTrue(AV.exists(tmpdir, "dummy", idf2, "layer1.0.conv1"))
def test_av_save_two_layers(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
av_0 = torch.randn(64, 16)
AV.save(tmpdir, "dummy", DEFAULT_IDENTIFIER, "layer1.0.conv1",
av_0, "0")
self.assertTrue(
AV.exists(tmpdir, "dummy", DEFAULT_IDENTIFIER,
"layer1.0.conv1"))
self.assertFalse(
AV.exists(tmpdir, "dummy", DEFAULT_IDENTIFIER,
"layer1.0.conv2"))
# experimenting with adding to another layer
av_1 = torch.randn(64, 16)
AV.save(tmpdir, "dummy", DEFAULT_IDENTIFIER, "layer1.0.conv2",
av_1, "0")
self.assertTrue(
AV.exists(tmpdir, "dummy", DEFAULT_IDENTIFIER,
"layer1.0.conv2"))
def test_TCAV_generate_all_activations(self) -> None:
def forward_hook_wrapper(expected_act: Tensor):
def forward_hook(module, inp, out=None):
out = torch.reshape(out, (out.shape[0], -1))
self.assertEqual(out.detach().shape[1:],
expected_act.shape[1:])
return forward_hook
with tempfile.TemporaryDirectory() as tmpdirname:
layers = ["conv1", "conv2", "fc1", "fc2"]
tcav, concept_dict = init_TCAV(tmpdirname,
CustomClassifier(),
layers=layers)
tcav.concepts = set(concept_dict.values())
# generating all activations for given layers and concepts
tcav.generate_all_activations()
# verify that all activations exist and have correct shapes
for layer in layers:
for _, concept in concept_dict.items():
self.assertTrue(
AV.exists(tmpdirname, "default_model_id",
concept.identifier, layer))
concept_meta: Dict[int, int] = defaultdict(int)
for _, concept in concept_dict.items():
activations = AV.load(tmpdirname, "default_model_id",
concept.identifier, layer)
def batch_collate(batch):
return torch.cat(batch)
self.assertTrue(concept.data_iter is not None)
assert not (activations is None)
for activation in cast(
Iterable,
DataLoader(activations, collate_fn=batch_collate)):
concept_meta[concept.id] += activation.shape[0]
layer_module = _get_module_from_name(tcav.model, layer)
for data in cast(Iterable, concept.data_iter):
hook = layer_module.register_forward_hook(
forward_hook_wrapper(activation))
tcav.model(data)
hook.remove()
# asserting the length of entire dataset for each concept
for concept_meta_i in concept_meta.values():
self.assertEqual(concept_meta_i, 100)
def save_and_assert_batch(layer_path, total_num_batches, batch,
n_batch_name):
# save n-th batch and verify the number of saved batches
AV.save(
tmpdir,
model_id,
DEFAULT_IDENTIFIER,
"layer1.0.conv1",
batch,
n_batch_name,
)
self.assertEqual(
len(glob.glob("/".join([layer_path, "*.pt"]))),
total_num_batches,
)
self.assertTrue(
AV.exists(tmpdir, model_id, DEFAULT_IDENTIFIER,
"layer1.0.conv1", n_batch_name))
def test_model_ids_in_tcav(self, ) -> None:
# creating concepts and mapping between concepts and their names
concepts_dict = create_concepts()
# defining experimental sets of different length
experimental_set_list = [["striped", "random"], ["dotted", "random"]]
experimental_sets = self._create_experimental_sets(
experimental_set_list, concepts_dict)
model = BasicModel_ConvNet()
model.eval()
layer = "conv2"
inputs = 100 * get_inputs_tensor()
with tempfile.TemporaryDirectory() as tmpdirname:
tcav1 = TCAV(
model,
layer,
model_id="my_basic_model1",
classifier=CustomClassifier(),
save_path=tmpdirname,
)
interpret1 = tcav1.interpret(inputs,
experimental_sets=experimental_sets,
target=0)
tcav2 = TCAV(
model,
layer,
model_id="my_basic_model2",
classifier=CustomClassifier(),
save_path=tmpdirname,
)
interpret2 = tcav2.interpret(inputs,
experimental_sets=experimental_sets,
target=0)
# testing that different folders were created for two different
# ids of the model
self.assertTrue(
AV.exists(
tmpdirname,
"my_basic_model1",
concepts_dict["striped"].identifier,
layer,
))
self.assertTrue(
AV.exists(
tmpdirname,
"my_basic_model2",
concepts_dict["striped"].identifier,
layer,
))
for interpret1_elem, interpret2_elem in zip(
interpret1, interpret2):
for interpret1_sub_elem, interpret2_sub_elem in zip(
interpret1[interpret1_elem],
interpret2[interpret2_elem]):
assertTensorAlmostEqual(
self,
interpret1[interpret1_elem][interpret1_sub_elem]
["sign_count"],
interpret2[interpret2_elem][interpret2_sub_elem]
["sign_count"],
0.0,
)
assertTensorAlmostEqual(
self,
interpret1[interpret1_elem][interpret1_sub_elem]
["magnitude"],
interpret2[interpret2_elem][interpret2_sub_elem]
["magnitude"],
0.0,
)
self.assertEqual(interpret1_sub_elem, interpret2_sub_elem)
self.assertEqual(interpret1_elem, interpret2_elem)