def test_binary_mcc():
Y_TRUE = [1, 1, 1, 0]
Y_PRED = [1, 0, 1, 1]
MCC_GOLD = -0.3333333333333333
cm = ConfusionMatrix([0, 1])
cm.add_batch(Y_TRUE, Y_PRED)
np.testing.assert_allclose(cm.get_mcc(), MCC_GOLD, TOL)
def _test(self, loader, **kwargs):
"""Test an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train.
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
:return: Metrics
"""
if self.ema:
self.sess.run(self.ema_load)
use_dataset = kwargs.get('dataset', True)
cm = ConfusionMatrix(self.model.labels)
steps = len(loader)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
for i, batch_dict in enumerate(pg(loader)):
y = batch_dict['y']
if use_dataset:
guess, lossv = self.sess.run([self.model.best, self.test_loss])
else:
feed_dict = self.model.make_input(batch_dict, False)
guess, lossv = self.sess.run([self.model.best, self.test_loss],
feed_dict=feed_dict)
batchsz = len(guess)
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, guess)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
def _test(self, loader, steps=0, **kwargs):
"""Test an epoch of data using either the input loader or using `tf.dataset`
In non-`tf.dataset` mode, we cycle the loader data feed, and pull a batch and feed it to the feed dict
When we use `tf.dataset`s under the hood, this function simply uses the loader to know how many steps
to train.
:param loader: A data feed
:param kwargs: See below
:Keyword Arguments:
* *dataset* (`bool`) Set to `True` if using `tf.dataset`s, defaults to `True`
* *reporting_fns* (`list`) A list of reporting hooks to use
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
:return: Metrics
"""
cm = ConfusionMatrix(self.model.labels)
total_loss = 0
total_norm = 0
verbose = kwargs.get("verbose", None)
pg = create_progress_bar(steps)
SET_TRAIN_FLAG(False)
for features, y in pg(loader):
logits = self.model(features)
y_ = tf.argmax(logits, axis=1, output_type=tf.int32)
cm.add_batch(y, y_)
lossv = tf.compat.v1.losses.sparse_softmax_cross_entropy(
labels=y, logits=logits).numpy()
batchsz = int(y.shape[0])
assert len(y_) == batchsz
total_loss += lossv * batchsz
total_norm += batchsz
cm.add_batch(y, y_)
metrics = cm.get_all_metrics()
metrics['avg_loss'] = total_loss / float(total_norm)
verbose_output(verbose, cm)
return metrics
return loss_value
for epoch in range(num_epochs):
loss_acc = 0.
step = 0
start = time.time()
for x, y in train_set.get_input(training=True):
loss_value = train_step(optimizer, model, x, y)
loss_acc += loss_value
step += 1
print('training time {}'.format(time.time() - start))
mean_loss = loss_acc / step
print('Training Loss {}'.format(mean_loss))
cm = ConfusionMatrix(['0', '1'])
for x, y in valid_set.get_input():
y_ = np.argmax(to_device(model(x)), axis=1)
cm.add_batch(y, y_)
print(cm)
print(cm.get_all_metrics())
print('FINAL')
cm = ConfusionMatrix(['0', '1'])
for x, y in test_set.get_input():
y_ = tf.argmax(to_device(model(x)), axis=1, output_type=tf.int32)
cm.add_batch(y, y_)
print(cm)
print(cm.get_all_metrics())