def setUp(self):
self.acts = np.array([[0, 1., 2.]])
self.examples = [None, None, None]
self.concepts = ['c1', 'c2']
self.target = 't1'
self.class_id = 0
self.bottleneck = 'bn'
self.cav_dir = None
self.hparams = {'model_type': 'linear', 'alpha': .01}
self.cav = CAV(self.concepts, self.bottleneck, self.hparams)
self.cav.cavs = [[
1.,
2.,
3.,
]]
self.activation_generator = None
self.mymodel = TcavTest_model()
self.act_gen = TcavTest_ActGen(self.mymodel)
self.random_counterpart = 'random500_1'
self.mytcav = TCAV(None, self.target, self.concepts, [self.bottleneck],
self.act_gen, [self.hparams['alpha']])
self.mytcav_random_counterpart = TCAV(None, self.target, self.concepts,
[self.bottleneck], self.act_gen,
[self.hparams['alpha']],
self.random_counterpart)
def setUp(self):
self.acts = np.array([[0, 1., 2.]])
self.concepts = ['c1', 'c2']
self.target = 't1'
self.class_id = 0
self.bottleneck = 'bn'
self.cav_dir = None
self.hparams = tf.contrib.training.HParams(model_type='linear',
alpha=.01)
self.cav = CAV(self.concepts, self.bottleneck, self.hparams)
self.cav.cavs = [[
1.,
2.,
3.,
]]
self.activation_generator = None
self.mymodel = TcavTest_model()
self.act_gen = TcavTest_ActGen(self.mymodel)
self.random_counterpart = 'random500_1'
self.mytcav = TCAV(None, self.target, self.concepts, [self.bottleneck],
self.act_gen, [self.hparams.alpha])
self.mytcav_random_counterpart = TCAV(None, self.target, self.concepts,
[self.bottleneck], self.act_gen,
[self.hparams.alpha],
self.random_counterpart)
def test_get_direction_dir_sign(self):
self.assertFalse(TCAV.get_direction_dir_sign(self.mymodel,
self.acts,
self.cav,
self.concepts[0],
self.class_id,
None))
def test_get_directional_dir(self):
directional_dirs = TCAV.get_directional_dir(self.mymodel,
self.target,
self.concepts[0],
self.cav,
self.acts,
self.examples)
self.assertAlmostEqual(8., directional_dirs[0])
def test_compute_tcav_score(self):
score = TCAV.compute_tcav_score(self.mymodel,
self.target,
self.concepts[0],
self.cav,
self.acts,
run_parallel=False)
self.assertAlmostEqual(0., score)
def test__process_what_to_run_expand_relative_tcav(self):
# _process_what_to_run_expand stores results to all_concepts,
# and pairs_to_test.
# test when concepts and random_concepts contain the same elements
concepts_relative = ['c1', 'c2', 'c3']
my_relative_tcav = TCAV(None,
self.target,
concepts_relative, [self.bottleneck],
self.act_gen, [self.hparams['alpha']],
random_concepts=concepts_relative)
self.mytcav_random_counterpart._process_what_to_run_expand(
num_random_exp=2, random_concepts=concepts_relative)
self.assertEqual(sorted(my_relative_tcav.all_concepts),
sorted(['t1', 'c1', 'c2', 'c3']))
self.assertEqual(
sorted(my_relative_tcav.pairs_to_test),
sorted([
('t1', ['c1', 'c2']),
('t1', ['c1', 'c3']),
('t1', ['c2', 'c1']),
('t1', ['c2', 'c3']),
('t1', ['c3', 'c1']),
('t1', ['c3', 'c2']),
]))
class TcavTest(googletest.TestCase):
def setUp(self):
self.acts = np.array([[0, 1., 2.]])
self.examples = [None, None, None]
self.concepts = ['c1', 'c2']
self.target = 't1'
self.class_id = 0
self.bottleneck = 'bn'
self.cav_dir = None
self.hparams = {'model_type': 'linear', 'alpha': .01}
self.cav = CAV(self.concepts, self.bottleneck, self.hparams)
self.cav.cavs = [[
1.,
2.,
3.,
]]
self.activation_generator = None
self.mymodel = TcavTest_model()
self.act_gen = TcavTest_ActGen(self.mymodel)
self.random_counterpart = 'random500_1'
self.mytcav = TCAV(None, self.target, self.concepts, [self.bottleneck],
self.act_gen, [self.hparams['alpha']])
self.mytcav_random_counterpart = TCAV(None, self.target, self.concepts,
[self.bottleneck], self.act_gen,
[self.hparams['alpha']],
self.random_counterpart)
def test_get_direction_dir_sign(self):
self.assertFalse(
TCAV.get_direction_dir_sign(self.mymodel, self.acts, self.cav,
self.concepts[0], self.class_id, None))
def test_compute_tcav_score(self):
score = TCAV.compute_tcav_score(self.mymodel,
self.target,
self.concepts[0],
self.cav,
self.acts,
self.examples,
run_parallel=False)
self.assertAlmostEqual(0., score)
def test_get_directional_dir(self):
directional_dirs = TCAV.get_directional_dir(self.mymodel, self.target,
self.concepts[0], self.cav,
self.acts, self.examples)
self.assertAlmostEqual(8., directional_dirs[0])
def test__run_single_set(self):
"""TODO(beenkim) not sure how to test this yet.
"""
pass
def test__process_what_to_run_expand(self):
# _process_what_to_run_expand stores results to all_concepts,
# and pairs_to_test.
self.mytcav._process_what_to_run_expand(num_random_exp=2)
self.assertEqual(
sorted(self.mytcav.all_concepts),
sorted(['t1', 'c1', 'c2', 'random500_0', 'random500_1']))
self.assertEqual(
sorted(self.mytcav.pairs_to_test),
sorted([('t1', ['c1',
'random500_0']), ('t1', ['c1', 'random500_1']),
('t1', ['c2', 'random500_0']),
('t1', ['c2', 'random500_1']),
('t1', ['random500_0', 'random500_1']),
('t1', ['random500_1', 'random500_0'])]))
def test__process_what_to_run_expand_random_counterpart(self):
# _process_what_to_run_expand stores results to all_concepts,
# and pairs_to_test.
# test when random_counterpart is supplied
self.mytcav_random_counterpart._process_what_to_run_expand(
num_random_exp=2)
self.assertEqual(
sorted(self.mytcav_random_counterpart.all_concepts),
sorted([
't1', 'c1', 'c2', 'random500_0', 'random500_1', 'random500_2'
]))
self.assertEqual(
sorted(self.mytcav_random_counterpart.pairs_to_test),
sorted([('t1', ['c1',
'random500_0']), ('t1', ['c1', 'random500_2']),
('t1', ['c2', 'random500_0']),
('t1', ['c2', 'random500_2']),
('t1', ['random500_1', 'random500_0']),
('t1', ['random500_1', 'random500_2'])]))
def test__process_what_to_run_expand_specify_dirs(self):
# _process_what_to_run_expand stores results to all_concepts,
# and pairs_to_test.
self.mytcav._process_what_to_run_expand(
num_random_exp=2, random_concepts=['random_dir1', 'random_dir2'])
self.assertEqual(
sorted(self.mytcav.all_concepts),
sorted(['t1', 'c1', 'c2', 'random_dir1', 'random_dir2']))
self.assertEqual(
sorted(self.mytcav.pairs_to_test),
sorted([('t1', ['c1',
'random_dir1']), ('t1', ['c1', 'random_dir2']),
('t1', ['c2', 'random_dir1']),
('t1', ['c2', 'random_dir2']),
('t1', ['random_dir1', 'random_dir2']),
('t1', ['random_dir2', 'random_dir1'])]))
def test__process_what_to_run_expand_specify_dirs_random_concepts(self):
# _process_what_to_run_expand stores results to all_concepts,
# and pairs_to_test.
# test when random_counterpart is supplied
self.mytcav_random_counterpart._process_what_to_run_expand(
num_random_exp=2, random_concepts=['random_dir1', 'random_dir2'])
self.assertEqual(
sorted(self.mytcav_random_counterpart.all_concepts),
sorted([
't1', 'c1', 'c2', 'random500_1', 'random_dir1', 'random_dir2'
]))
self.assertEqual(
sorted(self.mytcav_random_counterpart.pairs_to_test),
sorted([('t1', ['c1',
'random_dir1']), ('t1', ['c1', 'random_dir2']),
('t1', ['c2', 'random_dir1']),
('t1', ['c2', 'random_dir2']),
('t1', ['random500_1', 'random_dir1']),
('t1', ['random500_1', 'random_dir2'])]))
def test__process_what_to_run_expand_relative_tcav(self):
# _process_what_to_run_expand stores results to all_concepts,
# and pairs_to_test.
# test when concepts and random_concepts contain the same elements
concepts_relative = ['c1', 'c2', 'c3']
my_relative_tcav = TCAV(None,
self.target,
concepts_relative, [self.bottleneck],
self.act_gen, [self.hparams['alpha']],
random_concepts=concepts_relative)
self.mytcav_random_counterpart._process_what_to_run_expand(
num_random_exp=2, random_concepts=concepts_relative)
self.assertEqual(sorted(my_relative_tcav.all_concepts),
sorted(['t1', 'c1', 'c2', 'c3']))
self.assertEqual(
sorted(my_relative_tcav.pairs_to_test),
sorted([
('t1', ['c1', 'c2']),
('t1', ['c1', 'c3']),
('t1', ['c2', 'c1']),
('t1', ['c2', 'c3']),
('t1', ['c3', 'c1']),
('t1', ['c3', 'c2']),
]))
def test_get_params(self):
"""Check if the first param was correct.
"""
params = self.mytcav.get_params()
self.assertEqual(params[0].bottleneck, 'bn')
self.assertEqual(params[0].concepts, ['c1', 'random500_0'])
self.assertEqual(params[0].target_class, 't1')
self.assertEqual(params[0].activation_generator, self.act_gen)
self.assertEqual(params[0].cav_dir, self.cav_dir)
self.assertEqual(params[0].alpha, self.hparams['alpha'])
self.assertEqual(params[0].model, self.mymodel)
def run_tcav(model, dataset, previous_tcav_dict=None):
dataset_name = dataset.dataset_name
id_to_labels = dataset.id_to_label
model_to_run = 'inception_v3'
tcav_dir = "models/tensorflow_inception_v3_tcav_temps"
# where activations are stored (only if your act_gen_wrapper does so)
activation_dir = os.path.join(tcav_dir, 'activations/')
# where CAVs are stored.
# You can say None if you don't wish to store any.
cav_dir = os.path.join(tcav_dir, 'cavs/')
concept_directory = "datasets/tcav_concepts"
target_directory = "datasets/image_ILSVRC2012_validation"
bottlenecks = ['Mixed_5d', 'Mixed_7c'] # @param
utils.make_dir_if_not_exists(activation_dir)
utils.make_dir_if_not_exists(cav_dir)
# this is a regularizer penalty parameter for linear classifier to get CAVs.
alphas = [0.1]
# a folder that random images are stored
random_counterpart = 'random_images'
#targets = random.sample(id_to_labels.keys(), 50)
targets = [286, 370, 757, 595, 147, 108, 478, 517, 334, 173, 948, 727, 23]
if -1 in targets:
targets.remove(-1)
print(targets)
concepts = [
dI for dI in os.listdir(concept_directory)
if os.path.isdir(os.path.join(concept_directory, dI))
and "random" not in dI and "." not in dI
]
the_model = TCAVInceptionWrapperSlim(model.session, model, id_to_labels)
act_generator = act_gen.ImageActivationGenerator(
the_model,
concept_directory,
activation_dir,
max_examples=100,
target_dir=target_directory,
label_to_element_dict=dataset.label_to_elements)
tf.logging.set_verbosity(0)
if previous_tcav_dict == None:
tcav_dict = previous_tcav_dict
else:
tcav_dict = {}
for target in targets:
mytcav = TCAV(model.session,
target,
concepts,
bottlenecks,
act_generator,
alphas,
random_counterpart,
cav_dir=cav_dir,
num_random_exp=19,
use_numeric_class_label=True)
results = mytcav.run(run_parallel=True, num_workers=5)
tcav_dict = utils.print_results(results,
class_id=target,
result_dict=tcav_dict)
tcav_file_path = os.path.join(
"models",
dataset_name + model.model_name + '-tcavscores-2' + '.pkl')
with open(tcav_file_path, 'wb') as f:
pickle.dump(tcav_dict, f, pickle.HIGHEST_PROTOCOL)
return tcav_dict
def run_tcav():
model = get_model_list("../../models/")[0]
dataset = get_dataset_list("../../datasets")[0]
dataset_name = dataset.dataset_name
id_to_labels = dataset.id_to_label
model_to_run = 'inception_v3'
tcav_dir = "../../models/tensorflow_inception_v3"
# where activations are stored (only if your act_gen_wrapper does so)
activation_dir = os.path.join(tcav_dir, 'activations/')
# where CAVs are stored.
# You can say None if you don't wish to store any.
cav_dir = os.path.join(tcav_dir, 'cavs/')
concept_directory = "../../datasets/tcav_concepts"
target_directory = "../../datasets/targets"
bottlenecks = ['Mixed_5d'] # @param
utils.make_dir_if_not_exists(activation_dir)
utils.make_dir_if_not_exists(cav_dir)
# this is a regularizer penalty parameter for linear classifier to get CAVs.
alphas = [0.1]
# a folder that random images are stored
random_counterpart = 'random_images'
targets = ['zebra']
concepts = ["dotted", "striped", "zigzagged", "irregular pattern", "gradient", "single color"]
#crawl images for concepts and target class
for concept in concepts:
if not os.path.isdir(os.path.join(concept_directory, concept)):
image_crawler.crawl_images(concept_directory, concept, N=50)
# if not os.path.isdir(os.path.join(concept_directory, random_counterpart)):
# image_crawler.crawl_images(concept_directory, 'image', N=500)
for target in targets:
if not os.path.isdir(os.path.join(target_directory, target)):
image_crawler.crawl_images(target_directory, target, N=50)
the_model = cm.InceptionV3Wrapper_custom(model.session,
model,
id_to_labels)
act_generator = act_gen.ImageActivationGenerator(the_model, concept_directory, activation_dir, max_examples=100)
tf.logging.set_verbosity(0)
tcav_dict = {}
for target in targets:
mytcav = TCAV(model.session,
target,
concepts,
bottlenecks,
act_generator,
alphas,
random_counterpart,
cav_dir=cav_dir,
num_random_exp=5)
results = mytcav.run()
# we have to subtract 1 from the target class, as it corresponds with our ground truth labels,
# internally the network outputs are shifted by one, as 0 represents the background class instead of -1
summary = utils.print_results(results, class_id=the_model.label_to_id(target)-1, result_dict=tcav_dict)
tcav_file_path = os.path.join(model.model_path, dataset_name + model.model_name + '-tcavscores' + '.pkl')
with open(tcav_file_path, 'wb') as f:
pickle.dump(tcav_dict, f, pickle.HIGHEST_PROTOCOL)
def run_tcav(model, dataset, previous_tcav_dict=None):
'''
:param model:
:param dataset:
:param previous_tcav_dict:
:return:
'''
dataset_name = dataset.dataset_name
id_to_labels = dataset.id_to_label
tcav_dir = "models/tensorflow_inception_v3_tcav_temps"
# where activations are stored (only if your act_gen_wrapper does so)
activation_dir = os.path.join(tcav_dir, 'activations/')
# where CAVs are stored.
# You can say None if you don't wish to store any.
cav_dir = os.path.join(tcav_dir, 'cavs/')
concept_directory = "datasets/tcav_concepts"
target_directory = "datasets/image_ILSVRC2012_validation"
bottlenecks = ['Mixed_5d', 'Mixed_7c'] # @param
create_tcav_dirs(
dataset=dataset,
model=model,
concept_dir=concept_directory,
tcav_dir=tcav_dir,
nr_of_random_experiments=12,
concepts=[
"glossy surface", "matt surface", "uneven texture",
"wrinkled object", "rectangle pattern", "transparent object",
"translucent", "rectangular", "round", "triangular",
"complex shape", "spiky surface", "tree", "flower", "plant",
"grass,moss", "machine", "machine device", "architecture",
"clothes", "animal fur", "fabric,clothes", "inside lighting",
"nature", "beach", "underwater", "sky"
],
nr_imgs_per_concept=50,
nr_random_images=500)
# this is a regularizer penalty parameter for linear classifier to get CAVs.
alphas = [0.1]
# a folder that random images are stored
random_counterpart = 'random_images'
# targets = random.sample(id_to_labels.keys(), 10)
targets = [65, 970, 230, 809, 516, 57, 334, 415, 674, 332, 109]
if -1 in targets:
targets.remove(-1)
print(targets)
concepts = [
dI for dI in os.listdir(concept_directory)
if os.path.isdir(os.path.join(concept_directory, dI))
and "random" not in dI and "." not in dI
]
the_model = TCAVInceptionWrapperSlim(model.session, model, id_to_labels)
act_generator = act_gen.ImageActivationGenerator(
the_model,
concept_directory,
activation_dir,
max_examples=100,
target_dir=target_directory,
label_to_element_dict=dataset.label_to_elements)
tf.logging.set_verbosity(0)
if previous_tcav_dict == None:
tcav_dict = previous_tcav_dict
else:
tcav_dict = {}
for target in targets:
print("create tcavs for:", target)
mytcav = TCAV(model.session,
target,
concepts,
bottlenecks,
act_generator,
alphas,
random_counterpart,
cav_dir=cav_dir,
num_random_exp=19,
use_numeric_class_label=True)
print('Run TCAV creation')
results = mytcav.run(run_parallel=True, num_workers=5)
print('Finished tcav creation')
tcav_dict = utils.print_results(results,
class_id=target,
result_dict=tcav_dict)
print('Saved dict')
tcav_file_path = os.path.join(
model.model_path,
dataset_name + model.model_name + '-tcavscores' + '.pkl')
with open(tcav_file_path, 'wb') as f:
pickle.dump(tcav_dict, f, pickle.HIGHEST_PROTOCOL)
print('Saved to file')
return tcav_dict