def __classification_accuracy(self, sess, iter_init, idx, y_ph=None):
"""
:param sess: TensorFlow session
:param iter_init: TensorFlow data iterator initializer associated
:param idx: insertion index (i.e. epoch - 1)
:param y_ph: TensorFlow placeholder for unseen labels
:return: None
"""
if self.perf is None or y_ph is None:
return
# initialize results
y = np.zeros([0, 1])
y_hats = [np.zeros([0, 1])] * self.num_B_sub_heads
# initialize unsupervised data iterator
sess.run(iter_init)
# loop over the batches within the unsupervised data iterator
print('Evaluating classification accuracy... ')
while True:
try:
# grab the results
results = sess.run([self.y_hats, y_ph],
feed_dict={self.is_training: False})
# load metrics
for i in range(self.num_B_sub_heads):
y_hats[i] = np.concatenate(
(y_hats[i], np.expand_dims(results[0][i], axis=1)))
if y_ph is not None:
y = np.concatenate((y, np.expand_dims(results[1], axis=1)))
# _, ax = plt.subplots(2, 10)
# i_rand = np.random.choice(results[3].shape[0], 10)
# for i in range(10):
# ax[0, i].imshow(results[3][i_rand[i]][:, :, 0], origin='upper', vmin=0, vmax=1)
# ax[0, i].set_xticks([])
# ax[0, i].set_yticks([])
# ax[1, i].imshow(results[4][i_rand[i]][:, :, 0], origin='upper', vmin=0, vmax=1)
# ax[1, i].set_xticks([])
# ax[1, i].set_yticks([])
# plt.show()
# iterator will throw this error when its out of data
except tf.errors.OutOfRangeError:
break
# compute classification accuracy
if y_ph is not None:
class_errors = [
unsupervised_labels(y, y_hats[i], self.k_B, self.k_B)
for i in range(self.num_B_sub_heads)
]
self.perf['class_err_min'][idx] = np.min(class_errors)
self.perf['class_err_avg'][idx] = np.mean(class_errors)
self.perf['class_err_max'][idx] = np.max(class_errors)
# metrics are done
print('Done')
def eval(self):
"""Evaluate the accuracy of the current model weights
"""
y_pred = self._model.predict(self.x_test)
print("")
# accuracy per head
for head in range(self.args.heads):
if self.args.heads == 1:
y_head = y_pred
else:
y_head = y_pred[head]
y_head = np.argmax(y_head, axis=1)
accuracy = unsupervised_labels(list(self.y_test),
list(y_head),
self.n_labels,
self.n_labels)
info = "Head %d accuracy: %0.2f%%"
if self.accuracy > 0:
info += ", Old best accuracy: %0.2f%%"
data = (head, accuracy, self.accuracy)
else:
data = (head, accuracy)
print(info % data)
# if accuracy improves during training,
# save the model weights on a file
if accuracy > self.accuracy \
and self.args.save_weights is not None:
self.accuracy = accuracy
folder = self.args.save_dir
os.makedirs(folder, exist_ok=True)
path = os.path.join(folder, self.args.save_weights)
print("Saving weights... ", path)
self._model.save_weights(path)
def eval(self):
"""
@brief Evaluate the accuracy of the current model weights
"""
y_pred = self._model.predict(self.x_test)
# accuracy per head
for head in range(self._heads):
if self._heads == 1:
y_head = y_pred
else:
y_head = y_pred[head]
y_head = np.argmax(y_head, axis=1)
accuracy = unsupervised_labels(list(self.y_test), list(y_head),
self._z_dimension,
self._z_dimension)
info = "Head %d accuracy: %0.2f%%"
if self._accuracy > 0:
info += ", Old best accuracy: %0.2f%%"
data = (head, accuracy, self._accuracy)
else:
data = (head, accuracy)
print(info % data)