def metric_fn(predictions, labels, params=None):
images = params['inputs']
images = denormalize(images)
pred_prob = predictions['prob']
flat_pred_prob = tf.reshape(pred_prob, [-1])
pred_class = tf.to_float(tf.greater_equal(pred_prob, score_threshold))
# add summary for prediction results.
pred_images = pred_class * 255
vis_preds = tf.concat([
tf.cast(images, pred_class.dtype),
tf.tile(pred_images, multiples=[1, 1, 1, 3])
],
axis=2)
vis_preds = tf.cast(vis_preds, tf.uint8)
tf.summary.image('prediction', vis_preds, max_outputs=10)
labels = tf.reshape(labels, [-1])
pred_class = tf.reshape(pred_class, [-1])
miou = tf.metrics.mean_iou(labels, pred_class, num_classes=2)
p_iou = positive_iou(labels,
tf.reshape(pred_class, [-1]),
num_classes=2)
pr_curve('eval/prc',
tf.cast(labels, tf.bool),
flat_pred_prob,
num_thresholds=201)
return {'eval/miou': miou, 'eval/piou': p_iou}
def loss_fn(predictions, labels, params=None):
pred_logit = predictions['logit']
labels = tf.reshape(labels, [-1])
flat_logit = tf.reshape(pred_logit, [-1, num_classes])
if num_classes > 1:
onehot_labels = tf.one_hot(labels, num_classes)
else:
onehot_labels = tf.reshape(labels, [-1, 1])
bpn_weights = balance_positive_negative_weight(labels,
positive_weight=1.,
negative_weight=1.)
# bpn_weights = 1e-3 * bpn_weights
ce_loss = tf.losses.sigmoid_cross_entropy(onehot_labels,
flat_logit,
weights=bpn_weights[:, None])
pred_scores = tf.sigmoid(tf.reshape(flat_logit, [-1]))
# debug
# pred_scores = tf.Print(pred_scores, [pred_scores], message='pred_score = ')
# dice_loss_v = dice_loss(pred_scores, labels)
tf.summary.scalar('loss/cross_entropy_loss', ce_loss)
# tf.summary.scalar('loss/dice_loss', dice_loss_v)
# add metric for training
# mean iou
pred_class = tf.to_float(tf.greater_equal(pred_scores,
score_threshold))
miou = tf.metrics.mean_iou(labels,
tf.reshape(pred_class, [-1]),
num_classes=2)
miou_v = compute_mean_iou(None, miou[1])
tf.identity(miou_v, 'train_miou')
tf.summary.scalar('train/miou', miou_v)
# postive iou
p_iou = positive_iou(labels,
tf.reshape(pred_class, [-1]),
num_classes=2)
p_iou_v = compute_positive_iou(None, p_iou[1])
tf.identity(p_iou_v, 'train_piou')
tf.summary.scalar('train/piou', p_iou_v)
# negative iou
n_iou = negative_iou(labels,
tf.reshape(pred_class, [-1]),
num_classes=2)
n_iou_v = compute_negative_iou(None, n_iou[1])
tf.identity(n_iou_v, 'train_niou')
tf.summary.scalar('train/niou', n_iou_v)
# add pr curve
pr_curve('train/prc',
tf.cast(labels, tf.bool),
pred_scores,
num_thresholds=201)
return ce_loss
def _initialize_tensorboard(self):
tf.summary.scalar("accuracy", self.accuracy)
tf.summary.scalar("cost", self.cost)
# for cat in range(self.y.shape[1]): # this is apparently for all classes which I find weird but it works
summary_lib.pr_curve(
name='pr-curves',
predictions=self.pred[:, 1],
labels=tf.cast(self.y[:, 1], tf.bool),
num_thresholds=self.cf.num_thresholds,
display_name="pr-curve (train, test)",
description=
"Precision versus recall with respective cutoff. Hover over graph for details."
)
self.merged = tf.summary.merge_all()
def train_and_evaluate(classifier):
# Save a reference to the classifier to run predictions later
all_classifiers[classifier.model_dir] = classifier
classifier.train(input_fn=train_input_fn, steps=25000)
eval_results = classifier.evaluate(input_fn=eval_input_fn)
predictions = np.array([p['logistic'][0] for p in classifier.predict(input_fn=eval_input_fn)])
# Reset the graph to be able to reuse name scopes
tf.reset_default_graph()
# Add a PR summary in addition to the summaries that the classifier writes
pr = summary_lib.pr_curve('precision_recall', predictions=predictions, labels=y_test.astype(bool), num_thresholds=21)
with tf.Session() as sess:
writer = tf.summary.FileWriter(os.path.join(classifier.model_dir, 'eval'), sess.graph)
writer.add_summary(sess.run(pr), global_step=0)
writer.close()
x_len_train = np.array([min(len(x), sentence_size) for x in x_train_variable])
x_len_test = np.array([min(len(x), sentence_size) for x in x_test_variable])
embedding_matrix = load_glove_embeddings('data/glove.6B.50d.txt', word_index, vocab_size, embedding_size)
params = {'embedding_initializer': embedding_matrix}
lstm_classifier = tf.estimator.Estimator(model_fn=lstm_model_fn,
model_dir=os.path.join(model_dir, 'cnn_pretrained'),
params=params)
# Save a reference to the classifier to run predictions later
lstm_classifier.train(input_fn=train_input_fn(x_train, x_len_train, y_train, x_train_variable), steps=500)
eval_results = lstm_classifier.evaluate(input_fn=eval_input_fn(x_test, x_len_test, y_test))
predictions = np.array([p['logistic'][0] for p in lstm_classifier.predict(input_fn=eval_input_fn(x_test, x_len_test, y_test))])
tf.reset_default_graph()
pr = summary_lib.pr_curve('precision_recall', predictions=predictions, labels=y_test.astype(bool),
num_thresholds=21)
with tf.Session() as sess:
writer = tf.summary.FileWriter(os.path.join(lstm_classifier.model_dir, 'eval'), sess.graph)
writer.add_summary(sess.run(pr), global_step=0)
writer.close()
def print_predictions(sentences):
def text_to_index(sentence):
# Remove punctuation characters except for the apostrophe
translator = str.maketrans('', '', string.punctuation.replace("'", ''))
tokens = sentence.translate(translator).lower().split()
