def compute_metric(self, model, ds, num_testcases):
embeddings_list, labels_list = self.get_embeddings(
model, ds, num_testcases)
ret = compute_recall(
embeddings_list, labels_list, self.metric_conf['k'],
get_distance_function(self.conf['loss']['distance_function']))
return {'recall@{}'.format(k): score for k, score in ret.items()}
def testRecall2(self):
embeddings = tf.constant([
[1., 0.],
[4., 0.],
[0., 1.],
[1., 2.],
])
labels = tf.constant([1, 1, 2, 2], tf.int64)
ret = compute_recall(
[embeddings], [labels],
[1, 2, 3],
get_distance_function('euclidean_distance'))
self.assertEqual(ret, {1: 0.5, 2: 0.75, 3: 1.0})
ret = compute_recall(
[embeddings], [labels],
[1, 2, 3],
get_distance_function('cosine_similarity'))
self.assertEqual(ret, {1: 1.0, 2: 1.0, 3: 1.0})
def compute_metric(self, model, ds, num_testcases):
embeddings_list, labels_list = self.get_embeddings(
model, ds, num_testcases)
distance_function = get_distance_function(
self.conf['loss']['distance_function'])
embeddings = tf.concat(embeddings_list, axis=0)
labels = tf.concat(labels_list, axis=0)
label_map = defaultdict(list)
for index, label in enumerate(labels):
label_map[int(label)].append(index)
num_samples = self.metric_conf['num_samples']
positive_label_map = dict([(k, v) for k, v in label_map.items()
if len(v) >= 2])
positive_labels = random.choices(
population=list(positive_label_map.keys()),
weights=[len(v) for v in positive_label_map.values()],
k=num_samples,
)
positive_pairs = []
for positive_label in positive_labels:
a, b = random.sample(label_map[positive_label], 2)
positive_pairs.append((a, b))
first_list, second_list = zip(*positive_pairs)
positive_distances = compute_distances(embeddings, first_list,
second_list, distance_function)
negative_label_pairs = []
population = list(label_map.keys())
weights = [len(v) for v in label_map.values()]
for _ in range(num_samples):
while True:
a, b = random.choices(
population=population,
weights=weights,
k=2,
)
if a != b:
break
negative_label_pairs.append((a, b))
negative_pairs = []
for a, b in negative_label_pairs:
first = random.choice(label_map[a])
second = random.choice(label_map[b])
negative_pairs.append((first, second))
first_list, second_list = zip(*negative_pairs)
negative_distances = compute_distances(embeddings, first_list,
second_list, distance_function)
return float(
tf.reduce_sum(
tf.cast(positive_distances < negative_distances,
tf.float32))) / num_samples
def compute_metric(self, model, ds, num_testcases):
Metric.cache.clear()
data_loader = DataLoader.create(self.conf['dataset']['name'],
self.conf)
batch_design = BatchDesign.create(
self.metric_conf['batch_design']['name'], self.conf,
{'data_loader': data_loader})
pairs, labels, num_groups, num_examples = _make_pairs_dataset(
self.conf)
image_files, pair_indices = _flatten_image_files(pairs)
test_dataset, num_testcases = batch_design.create_dataset(
model,
image_files,
pair_indices,
self.metric_conf['batch_design'],
testing=True)
embeddings_list, pair_indices_list = self.get_embeddings(
model, test_dataset, num_testcases)
embedding_pairs = _make_embedding_pairs(embeddings_list)
distances = []
distance_function = get_distance_function(
self.conf['loss']['distance_function'])
for i, embedding_pair in enumerate(
tqdm(embedding_pairs, desc='lfw:distance',
dynamic_ncols=True)):
distance = compute_elementwise_distances(
tf.reshape(embedding_pair[0], [1, -1]),
tf.reshape(embedding_pair[1], [1, -1]), distance_function)
distances.append(float(distance))
accuracies = []
num_examples_per_group = num_examples * 2
num_total_examples = num_groups * num_examples_per_group
for i in trange(0,
num_total_examples,
num_examples_per_group,
desc='lfw:accuracy',
dynamic_ncols=True):
accuracy = _calculate_accuracy(
distances[i:i + num_examples_per_group],
labels[i:i + num_examples_per_group])
accuracies.append(accuracy)
Metric.cache.clear()
return {
'lfw_acc_avg': float(np.mean(accuracies)),
'lfw_acc_std': float(np.std(accuracies))
}
def testRecall(self):
embeddings = tf.constant([
[1., 0.],
[1., 0.],
[0., 1.],
[0., 1.],
])
labels = tf.constant([1, 1, 2, 2], tf.int64)
for parametrization in ['euclidean_distance', 'cosine_similarity']:
r = compute_recall([embeddings], [labels], [1], get_distance_function(parametrization))
self.assertEqual(r, {1: 1.0})
def testRecallWithSingleton(self):
embeddings = tf.constant([
[1., 0.],
[1., 0.],
[0., 1.],
[0., 1.],
[0., 1.],
])
labels = tf.constant([1, 1, 2, 3, 4], tf.int64)
r = compute_recall(
[embeddings], [labels],
[1],
get_distance_function('euclidean_distance'))
self.assertEqual(r, {1: 1.0})
def compute_metric(self, model, ds, num_testcases):
embeddings_list, labels_list = self.get_embeddings(
model, ds, num_testcases)
distance_function = get_distance_function(
self.conf['loss']['distance_function'])
embeddings = tf.concat(embeddings_list, axis=0)
labels = tf.concat(labels_list, axis=0)
label_map = defaultdict(list)
for index, label in enumerate(labels):
label_map[int(label)].append(index)
num_samples = self.metric_conf['num_samples']
positive_label_map = dict(
[(k, v) for k, v in label_map.items() if len(v) >= 2]
)
positive_labels = random.choices(
population=list(positive_label_map.keys()),
weights=[len(v) for v in positive_label_map.values()],
k=num_samples,
)
positive_pairs = []
for positive_label in positive_labels:
a, b = random.sample(label_map[positive_label], 2)
positive_pairs.append((a, b))
first_list, second_list = zip(*positive_pairs)
positive_distances = compute_distances(
embeddings, first_list, second_list, distance_function)
negative_labels = []
population = list(label_map.keys())
max_k = max(self.metric_conf['k'])
for _ in range(num_samples):
l = random.sample(population, max_k)
negative_labels.append(list(l))
negative_data = []
for label_list in negative_labels:
negative_data.append([random.choice(label_map[a]) for a in label_list])
negative_distances = compute_distances(
embeddings, tf.constant(first_list)[:, None], negative_data, distance_function)
ret = {}
for k in self.metric_conf['k']:
ret['vrf@{}'.format(k)] = float(tf.reduce_sum(tf.cast(
positive_distances < tf.reduce_min(negative_distances[:, :k], axis=1),
tf.float32
)) / num_samples)
return ret