def _process_what_to_run_expand(self,
num_random_exp=100,
random_concepts=None):
"""Get tuples of parameters to run TCAV with.
TCAV builds random concept to conduct statistical significance testing
againts the concept. To do this, we build many concept vectors, and many
random vectors. This function prepares runs by expanding parameters.
Args:
num_random_exp: number of random experiments to run to compare.
random_concepts: A list of names of random concepts for the random experiments
to draw from. Optional, if not provided, the names will be
random500_{i} for i in num_random_exp.
"""
target_concept_pairs = [(self.target, self.concepts)]
all_concepts_concepts, pairs_to_run_concepts = (
utils.process_what_to_run_expand(
utils.process_what_to_run_concepts(target_concept_pairs),
self.random_counterpart,
num_random_exp=num_random_exp,
random_concepts=random_concepts))
all_concepts_randoms, pairs_to_run_randoms = (
utils.process_what_to_run_expand(utils.process_what_to_run_randoms(
target_concept_pairs, self.random_counterpart),
self.random_counterpart,
num_random_exp=num_random_exp,
random_concepts=random_concepts))
self.all_concepts = list(
set(all_concepts_concepts + all_concepts_randoms))
self.pairs_to_test = pairs_to_run_concepts + pairs_to_run_randoms
def test_process_what_to_run_expand(self):
all_concepts, pairs_to_test = process_what_to_run_expand(
self.pair_to_test_one_concept, num_random_exp=2)
self.assertEqual(
sorted(all_concepts),
sorted(['t1', 'c1', 'c2', 'random500_1', 'random500_0']))
self.assertEqual(
sorted(pairs_to_test),
sorted([('t1', ['c1',
'random500_0']), ('t1', ['c1', 'random500_1']),
('t1', ['c2', 'random500_0']),
('t1', ['c2', 'random500_1'])]))
def test_process_what_to_run_expand_specify_dirs(self):
all_concepts, pairs_to_test = process_what_to_run_expand(
self.pair_to_test_one_concept,
num_random_exp=2,
random_concepts=['random_dir1', 'random_dir2'])
self.assertEqual(
sorted(all_concepts),
sorted(['t1', 'c1', 'c2', 'random_dir1', 'random_dir2']))
self.assertEqual(
sorted(pairs_to_test),
sorted([('t1', ['c1',
'random_dir1']), ('t1', ['c1', 'random_dir2']),
('t1', ['c2', 'random_dir1']),
('t1', ['c2', 'random_dir2'])]))
def _process_what_to_run_expand(self,
num_random_exp=100,
random_concepts=None):
"""Get tuples of parameters to run TCAV with.
TCAV builds random concept to conduct statistical significance testing
againts the concept. To do this, we build many concept vectors, and many
random vectors. This function prepares runs by expanding parameters.
Args:
num_random_exp: number of random experiments to run to compare.
random_concepts: A list of names of random concepts for the random experiments
to draw from. Optional, if not provided, the names will be
random500_{i} for i in num_random_exp.
"""
target_concept_pairs = [(self.target, self.concepts)]
# take away 1 random experiment if the random counterpart already in random concepts
# take away 1 random experiment if computing Relative TCAV
all_concepts_concepts, pairs_to_run_concepts = (
utils.process_what_to_run_expand(
utils.process_what_to_run_concepts(target_concept_pairs),
self.random_counterpart,
num_random_exp=num_random_exp -
(1 if random_concepts
and self.random_counterpart in random_concepts else 0) -
(1 if self.relative_tcav else 0),
random_concepts=random_concepts))
pairs_to_run_randoms = []
all_concepts_randoms = []
# ith random concept
def get_random_concept(i):
return (random_concepts[i]
if random_concepts else 'random500_{}'.format(i))
if self.random_counterpart is None:
# TODO random500_1 vs random500_0 is the same as 1 - (random500_0 vs random500_1)
for i in range(num_random_exp):
all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = (
utils.process_what_to_run_expand(
utils.process_what_to_run_randoms(
target_concept_pairs, get_random_concept(i)),
num_random_exp=num_random_exp - 1,
random_concepts=random_concepts))
pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp)
all_concepts_randoms.extend(all_concepts_randoms_tmp)
else:
# run only random_counterpart as the positve set for random experiments
all_concepts_randoms_tmp, pairs_to_run_randoms_tmp = (
utils.process_what_to_run_expand(
utils.process_what_to_run_randoms(target_concept_pairs,
self.random_counterpart),
self.random_counterpart,
num_random_exp=num_random_exp -
(1 if random_concepts
and self.random_counterpart in random_concepts else 0),
random_concepts=random_concepts))
pairs_to_run_randoms.extend(pairs_to_run_randoms_tmp)
all_concepts_randoms.extend(all_concepts_randoms_tmp)
self.all_concepts = list(
set(all_concepts_concepts + all_concepts_randoms))
self.pairs_to_test = pairs_to_run_concepts if self.relative_tcav else pairs_to_run_concepts + pairs_to_run_randoms
