def evaluate(sess, enqueue_op, image_paths_placeholder, labels_placeholder,
phase_train_placeholder, batch_size_placeholder,
control_placeholder, embeddings, labels, image_paths,
actual_issame, batch_size, nrof_folds, log_dir, step,
summary_writer, stat, epoch, distance_metric, subtract_mean,
use_flipped_images, use_fixed_image_standardization):
start_time = time.time()
# Run forward pass to calculate embeddings
print('Runnning forward pass on LFW images')
# Enqueue one epoch of image paths and labels
nrof_embeddings = len(
actual_issame) * 2 # nrof_pairs * nrof_images_per_pair
nrof_flips = 2 if use_flipped_images else 1
nrof_images = nrof_embeddings * nrof_flips
labels_array = np.expand_dims(np.arange(0, nrof_images), 1)
image_paths_array = np.expand_dims(
np.repeat(np.array(image_paths), nrof_flips), 1)
control_array = np.zeros_like(labels_array, np.int32)
if use_fixed_image_standardization:
control_array += np.ones_like(
labels_array) * facenet.FIXED_STANDARDIZATION
if use_flipped_images:
# Flip every second image
control_array += (labels_array % 2) * facenet.FLIP
sess.run(
enqueue_op, {
image_paths_placeholder: image_paths_array,
labels_placeholder: labels_array,
control_placeholder: control_array
})
embedding_size = int(embeddings.get_shape()[1])
assert nrof_images % batch_size == 0, 'The number of LFW images must be an integer multiple of the LFW batch size'
nrof_batches = nrof_images // batch_size
emb_array = np.zeros((nrof_images, embedding_size))
lab_array = np.zeros((nrof_images, ))
for i in range(nrof_batches):
feed_dict = {
phase_train_placeholder: False,
batch_size_placeholder: batch_size
}
emb, lab = sess.run([embeddings, labels], feed_dict=feed_dict)
lab_array[lab] = lab
emb_array[lab, :] = emb
if i % 10 == 9:
print('.', end='')
sys.stdout.flush()
print('')
embeddings = np.zeros((nrof_embeddings, embedding_size * nrof_flips))
if use_flipped_images:
# Concatenate embeddings for flipped and non flipped version of the images
embeddings[:, :embedding_size] = emb_array[0::2, :]
embeddings[:, embedding_size:] = emb_array[1::2, :]
else:
embeddings = emb_array
assert np.array_equal(
lab_array, np.arange(nrof_images)
) == True, 'Wrong labels used for evaluation, possibly caused by training examples left in the input pipeline'
_, _, accuracy, val, val_std, far = lfw.evaluate(
embeddings,
actual_issame,
nrof_folds=nrof_folds,
distance_metric=distance_metric,
subtract_mean=subtract_mean)
print('Accuracy: %2.5f+-%2.5f' % (np.mean(accuracy), np.std(accuracy)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' % (val, val_std, far))
lfw_time = time.time() - start_time
# Add validation loss and accuracy to summary
summary = tf.Summary()
#pylint: disable=maybe-no-member
summary.value.add(tag='lfw/accuracy', simple_value=np.mean(accuracy))
summary.value.add(tag='lfw/val_rate', simple_value=val)
summary.value.add(tag='time/lfw', simple_value=lfw_time)
summary_writer.add_summary(summary, step)
with open(os.path.join(log_dir, 'lfw_result.txt'), 'at') as f:
f.write('%d\t%.5f\t%.5f\n' % (step, np.mean(accuracy), val))
stat['lfw_accuracy'][epoch - 1] = np.mean(accuracy)
stat['lfw_valrate'][epoch - 1] = val
def evaluate(sess,
image_paths,
embeddings,
labels_batch,
image_paths_placeholder,
labels_placeholder,
batch_size_placeholder,
learning_rate_placeholder,
phase_train_placeholder,
enqueue_op,
actual_issame,
batch_size,
nrof_folds,
log_dir,
step,
summary_writer,
embedding_size,
threshold_to_test=None,
tag='lfw'):
result = {}
start_time = time.time()
# Run forward pass to calculate embeddings
print('Running forward pass on ' + tag + ' images: ', end='')
nrof_images = len(actual_issame) * 2
assert (len(image_paths) == nrof_images)
labels_array = np.reshape(np.arange(nrof_images), (-1, 4))
image_paths_array = np.reshape(np.expand_dims(np.array(image_paths), 1),
(-1, 4))
sess.run(
enqueue_op, {
image_paths_placeholder: image_paths_array,
labels_placeholder: labels_array
})
emb_array = np.zeros((nrof_images, embedding_size))
nrof_batches = int(np.ceil(nrof_images / batch_size))
label_check_array = np.zeros((nrof_images, ))
for i in xrange(nrof_batches):
batch_size = min(nrof_images - i * batch_size, batch_size)
emb, lab = sess.run(
[embeddings, labels_batch],
feed_dict={
batch_size_placeholder: batch_size,
learning_rate_placeholder: 0.0,
phase_train_placeholder: False
})
emb_array[lab, :] = emb
label_check_array[lab] = 1
print('%.3f' % (time.time() - start_time))
assert (np.all(label_check_array == 1))
tpr, fpr, accuracy, val, val_std, far, best_threshold, threshold_lowfar, tpr_lowfar, acc_lowfar = lfw.evaluate(
emb_array, actual_issame, nrof_folds=nrof_folds)
print('Accuracy: %1.3f+-%1.3f' % (np.mean(accuracy), np.std(accuracy)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' % (val, val_std, far))
lfw_time = time.time() - start_time
# Add validation loss and accuracy to summary
summary = tf.Summary()
# pylint: disable=maybe-no-member
summary.value.add(tag=tag + '/accuracy', simple_value=np.mean(accuracy))
summary.value.add(tag=tag + '/val_rate', simple_value=val)
summary.value.add(tag=tag + '/best_threshold', simple_value=best_threshold)
summary.value.add(tag=tag + '/val_rate_threshold',
simple_value=threshold_lowfar)
summary.value.add(tag='time/' + tag, simple_value=lfw_time)
summary_writer.add_summary(summary, step)
with open(os.path.join(log_dir, tag + '_result.txt'), 'at') as f:
f.write('%d\t%.5f\t%.5f\n' % (step, np.mean(accuracy), val))
result['accuracy'] = np.mean(accuracy)
result['val_rate'] = val
result['best_threshold'] = best_threshold
return result
def evaluate(config: config_reader.validation_config,
model: facenet.model_container,
sess,
image_paths,
actual_issame,
tag='eval'):
# Run forward pass to calculate embeddings
print('Runnning forward pass on ' + tag + ' images')
# Enqueue one epoch of image paths and labels
nrof_embeddings = len(image_paths) # nrof_pairs * nrof_images_per_pair
nrof_flips = 2 if config.use_flipped_images else 1
nrof_images = nrof_embeddings * nrof_flips
batch_size = 0
if nrof_images < 100:
batch_size = nrof_images
else:
# Try multiple batch size to fit number or images
for i in range(1, 101):
if nrof_images % i == 0:
batch_size = i
assert not batch_size == 0, 'Fitting batch size not found for {} images.'.format(nrof_images)
print('Batch size: {}'.format(batch_size))
labels_array = np.expand_dims(np.arange(0,nrof_images),1)
image_paths_array = np.expand_dims(np.repeat(np.array(image_paths),nrof_flips),1)
control_array = np.zeros_like(labels_array, np.int32)
if config.use_fixed_image_standardization:
control_array += np.ones_like(labels_array)*facenet.FIXED_STANDARDIZATION
if config.use_flipped_images:
# Flip every second image
control_array += (labels_array % 2)*facenet.FLIP
sess.run(model.enqueue_op, {model.image_paths_placeholder: image_paths_array, model.labels_placeholder: labels_array, model.control_placeholder: control_array})
embedding_size = int(model.embeddings.get_shape()[1])
assert nrof_images % batch_size == 0, 'The number of {} images ({}) must be an integer multiple of the batch size ({})'.format(tag, nrof_images, batch_size)
nrof_batches = nrof_images // batch_size
emb_array = np.zeros((nrof_images, embedding_size))
lab_array = np.zeros((nrof_images,))
for i in range(nrof_batches):
feed_dict = {model.phase_train_placeholder:False, model.batch_size_placeholder: batch_size}
emb, lab = sess.run([model.embeddings, model.labels_batch], feed_dict=feed_dict)
lab_array[lab] = lab
emb_array[lab, :] = emb
if i % 10 == 9:
print('.', end='')
sys.stdout.flush()
print('')
embeddings = np.zeros((nrof_embeddings, embedding_size*nrof_flips))
if config.use_flipped_images:
# Concatenate embeddings for flipped and non flipped version of the images
embeddings[:,:embedding_size] = emb_array[0::2,:]
embeddings[:,embedding_size:] = emb_array[1::2,:]
else:
embeddings = emb_array
assert np.array_equal(lab_array, np.arange(nrof_images))==True, 'Wrong labels used for evaluation, possibly caused by training examples left in the input pipeline'
tpr, fpr, accuracy, val, val_std, far, best_threshold, threshold_lowfar, tpr_lowfar, acc_lowfar = lfw.evaluate(embeddings,
actual_issame,
nrof_folds=config.nrof_folds,
distance_metric=config.distance_metric,
subtract_mean=config.subtract_mean)
print('Accuracy: %2.5f+-%2.5f' % (np.mean(accuracy), np.std(accuracy)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' % (val, val_std, far))
print('Best threshold: %1.3f' % best_threshold)
print('Threshold: %1.3f @ FAR=%2.5f' % (threshold_lowfar, far))
auc = metrics.auc(fpr, tpr)
print('Area Under Curve (AUC): %1.6f' % auc)
eer = brentq(lambda x: 1. - x - interpolate.interp1d(fpr, tpr)(x), 0., 1.)
print('Equal Error Rate (EER): %1.6f' % eer)