def _calculate_cav(self, c, r, bn, act_c, ow, directory=None):
"""Calculates a sinle cav for a concept and a one random counterpart.
Args:
c: conept name
r: random concept name
bn: the layer name
act_c: activation matrix of the concept in the 'bn' layer
ow: overwrite if CAV already exists
directory: to save the generated CAV
Returns:
The accuracy of the CAV
"""
if directory is None:
directory = self.cav_dir
act_r = self._random_concept_activations(bn, r)
cav_instance = cav.get_or_train_cav([c, r],
bn, {
c: {
bn: act_c
},
r: {
bn: act_r
}
},
cav_dir=directory,
overwrite=ow)
return cav_instance.accuracies['overall']
def test_get_or_train_cav_save_test(self):
cav_instance = get_or_train_cav(self.concepts,
self.bottleneck,
self.acts,
cav_dir=self.cav_dir,
cav_hparams=self.hparams)
# check values of some elements.
self.assertEqual(cav_instance.cavs[0][0], self.cav_vecs[0][0])
self.assertEqual(cav_instance.cavs[1][2], self.cav_vecs[1][2])
def _run_single_set(self, param, overwrite=False, run_parallel=False):
"""Run TCAV with provided for one set of (target, concepts).
Args:
param: parameters to run
overwrite: if True, overwrite any saved CAV files.
run_parallel: run this parallel.
Returns:
a dictionary of results (panda frame)
"""
bottleneck = param.bottleneck
concepts = param.concepts
target_class = param.target_class
activation_generator = param.activation_generator
alpha = param.alpha
mymodel = param.model
cav_dir = param.cav_dir
# first check if target class is in model.
tf.logging.info('running %s %s' % (target_class, concepts))
# Get acts
acts = activation_generator.process_and_load_activations(
[bottleneck], concepts + [target_class])
# Get CAVs
cav_hparams = CAV.default_hparams()
cav_hparams.alpha = alpha
cav_instance = get_or_train_cav(concepts,
bottleneck,
acts,
cav_dir=cav_dir,
cav_hparams=cav_hparams,
overwrite=overwrite)
# clean up
for c in concepts:
del acts[c]
# Hypo testing
a_cav_key = CAV.cav_key(concepts, bottleneck, cav_hparams.model_type,
cav_hparams.alpha)
target_class_for_compute_tcav_score = target_class
cav_concept = concepts[0]
i_up = self.compute_tcav_score(
mymodel,
target_class_for_compute_tcav_score,
cav_concept,
cav_instance,
acts[target_class][cav_instance.bottleneck],
activation_generator.get_examples_for_concept(target_class),
run_parallel=run_parallel)
val_directional_dirs = self.get_directional_dir(
mymodel, target_class_for_compute_tcav_score, cav_concept,
cav_instance, acts[target_class][cav_instance.bottleneck],
activation_generator.get_examples_for_concept(target_class))
result = {
'cav_key': a_cav_key,
'cav_concept': cav_concept,
'negative_concept': concepts[1],
'target_class': target_class,
'cav_accuracies': cav_instance.accuracies,
'i_up': i_up,
'val_directional_dirs_abs_mean':
np.mean(np.abs(val_directional_dirs)),
'val_directional_dirs_mean': np.mean(val_directional_dirs),
'val_directional_dirs_std': np.std(val_directional_dirs),
'val_directional_dirs': val_directional_dirs,
'note': 'alpha_%s ' % (alpha),
'alpha': alpha,
'bottleneck': bottleneck
}
del acts
return result
def _run_single_set(self, param):
"""Run TCAV with provided for one set of (target, concepts).
Args:
param: parameters to run
Returns:
a dictionary of results (panda frame)
"""
bottleneck = param.bottleneck
concepts = param.concepts
target_class = param.target_class
activation_generator = param.activation_generator
alpha = param.alpha
mymodel = param.model
cav_dir = param.cav_dir
# first check if target class is in model.
tf.logging.info('running %s %s' % (target_class, concepts))
print("running: target->", target_class, " concepts->", concepts)
# Get acts
if self.use_numeric_class_label:
# To correctly concatenate strings with the target class, we cast it into a string
in_target_class = str(
target_class) if self.use_numeric_class_label else target_class
acts = activation_generator.process_and_load_activations(
[bottleneck], concepts + [in_target_class], target_class)
else:
acts = activation_generator.process_and_load_activations(
[bottleneck], concepts + [in_target_class])
# Get CAVs
cav_hparams = CAV.default_hparams()
cav_hparams.alpha = alpha
cav_instance = get_or_train_cav(concepts,
bottleneck,
acts,
cav_dir=cav_dir,
cav_hparams=cav_hparams)
# clean up
for c in concepts:
del acts[c]
# Hypo testing
a_cav_key = CAV.cav_key(concepts, bottleneck, cav_hparams.model_type,
cav_hparams.alpha)
if self.use_numeric_class_label:
target_class_for_compute_tcav_score = target_class + 1
else:
target_class_for_compute_tcav_score = mymodel.label_to_id(
target_class) + 1
for cav_concept in concepts:
if cav_concept is self.random_counterpart or 'random500' not in cav_concept:
i_up = self.compute_tcav_score(
mymodel,
target_class_for_compute_tcav_score,
cav_concept,
cav_instance,
acts[str(target_class)][cav_instance.bottleneck],
)
val_directional_dirs = self.get_directional_dir(
mymodel, target_class_for_compute_tcav_score, cav_concept,
cav_instance,
acts[str(target_class)][cav_instance.bottleneck])
result = {
'cav_key':
a_cav_key,
'cav_concept':
cav_concept,
'target_class':
target_class,
'i_up':
i_up,
'val_directional_dirs_abs_mean':
np.mean(np.abs(val_directional_dirs)),
'val_directional_dirs_mean':
np.mean(val_directional_dirs),
'val_directional_dirs_std':
np.std(val_directional_dirs),
'note':
'alpha_%s ' % (alpha),
'alpha':
alpha,
'bottleneck':
bottleneck
}
del acts
return result
def _run_single_set(self, param, overwrite=False, run_parallel=False):
"""Run TCAV with provided for one set of (target, concepts).
Args:
param: parameters to run
overwrite: if True, overwrite any saved CAV files.
run_parallel: run this parallel.
Returns:
a dictionary of results (panda frame)
"""
bottleneck = param.bottleneck
concepts = param.concepts
target_class = param.target_class
activation_generator = param.activation_generator
alpha = param.alpha
mymodel = param.model
cav_dir = param.cav_dir
# first check if target class is in model.
tf.logging.info('running %s %s' % (target_class, concepts))
keyword = ''
is_keyword = False
if self.logit_grad:
keyword += ':logit_grad'
is_keyword = True
if self.grad_nomalize:
keyword += ':grad_nomalize'
is_keyword = True
if is_keyword:
keyword += ':'
if self.make_random and os.path.exists(
self.tcav_dir +
'{}:{}:{}:{}_{}{}'.format(bottleneck, target_class, alpha,
concepts[0], concepts[1], keyword)):
return None
# Get acts
acts = activation_generator.process_and_load_activations(
[bottleneck], concepts + [target_class])
# Get CAVs
cav_hparams = CAV.default_hparams()
cav_hparams.alpha = alpha
cav_instance = get_or_train_cav(concepts,
bottleneck,
acts,
cav_dir=cav_dir,
cav_hparams=cav_hparams,
overwrite=overwrite)
if self.true_cav == False:
# clean up
for c in concepts:
del acts[c]
# Hypo testing
a_cav_key = CAV.cav_key(concepts, bottleneck, cav_hparams.model_type,
cav_hparams.alpha)
target_class_for_compute_tcav_score = target_class
cav_concept = concepts[0]
#tmp = activation_generator.get_examples_for_concept(target_class)
i_up = self.compute_tcav_score(
mymodel,
target_class_for_compute_tcav_score,
cav_concept,
cav_instance,
acts[target_class][cav_instance.bottleneck],
activation_generator.get_examples_for_concept(target_class),
acts,
cav_instance.bottleneck,
activation_generator,
cav_dir,
self.true_cav,
self.logit_grad,
run_parallel=run_parallel)
val_directional_dirs = self.get_directional_dir_plus(
mymodel, target_class_for_compute_tcav_score, cav_concept,
cav_instance, acts[target_class][cav_instance.bottleneck],
activation_generator.get_examples_for_concept(target_class),
self.cav_dir, self.project_name, bottleneck, concepts[1], acts,
activation_generator, self.true_cav, self.logit_grad,
self.grad_nomalize, self.make_random)
result = {
'cav_key': a_cav_key,
'cav_concept': cav_concept,
'negative_concept': concepts[1],
'target_class': target_class,
'cav_accuracies': cav_instance.accuracies,
'i_up': i_up,
'val_directional_dirs': val_directional_dirs,
'alpha': alpha,
'bottleneck': bottleneck
}
del acts
if self.make_random and not os.path.exists(
self.tcav_dir +
'{}:{}:{}:{}_{}{}'.format(bottleneck, target_class, alpha,
concepts[0], concepts[1], keyword)):
pickle_dump(
result, self.tcav_dir +
'{}:{}:{}:{}_{}{}'.format(bottleneck, target_class, alpha,
concepts[0], concepts[1], keyword))
return result
