def metric_fn(predictions, labels, params=None): images = params['inputs'] images = denormalize(images) pred_prob = predictions['prob'] flat_pred_prob = tf.reshape(pred_prob, [-1]) pred_class = tf.to_float(tf.greater_equal(pred_prob, score_threshold)) # add summary for prediction results. pred_images = pred_class * 255 vis_preds = tf.concat([ tf.cast(images, pred_class.dtype), tf.tile(pred_images, multiples=[1, 1, 1, 3]) ], axis=2) vis_preds = tf.cast(vis_preds, tf.uint8) tf.summary.image('prediction', vis_preds, max_outputs=10) labels = tf.reshape(labels, [-1]) pred_class = tf.reshape(pred_class, [-1]) miou = tf.metrics.mean_iou(labels, pred_class, num_classes=2) p_iou = positive_iou(labels, tf.reshape(pred_class, [-1]), num_classes=2) pr_curve('eval/prc', tf.cast(labels, tf.bool), flat_pred_prob, num_thresholds=201) return {'eval/miou': miou, 'eval/piou': p_iou}
def loss_fn(predictions, labels, params=None): pred_logit = predictions['logit'] labels = tf.reshape(labels, [-1]) flat_logit = tf.reshape(pred_logit, [-1, num_classes]) if num_classes > 1: onehot_labels = tf.one_hot(labels, num_classes) else: onehot_labels = tf.reshape(labels, [-1, 1]) bpn_weights = balance_positive_negative_weight(labels, positive_weight=1., negative_weight=1.) # bpn_weights = 1e-3 * bpn_weights ce_loss = tf.losses.sigmoid_cross_entropy(onehot_labels, flat_logit, weights=bpn_weights[:, None]) pred_scores = tf.sigmoid(tf.reshape(flat_logit, [-1])) # debug # pred_scores = tf.Print(pred_scores, [pred_scores], message='pred_score = ') # dice_loss_v = dice_loss(pred_scores, labels) tf.summary.scalar('loss/cross_entropy_loss', ce_loss) # tf.summary.scalar('loss/dice_loss', dice_loss_v) # add metric for training # mean iou pred_class = tf.to_float(tf.greater_equal(pred_scores, score_threshold)) miou = tf.metrics.mean_iou(labels, tf.reshape(pred_class, [-1]), num_classes=2) miou_v = compute_mean_iou(None, miou[1]) tf.identity(miou_v, 'train_miou') tf.summary.scalar('train/miou', miou_v) # postive iou p_iou = positive_iou(labels, tf.reshape(pred_class, [-1]), num_classes=2) p_iou_v = compute_positive_iou(None, p_iou[1]) tf.identity(p_iou_v, 'train_piou') tf.summary.scalar('train/piou', p_iou_v) # negative iou n_iou = negative_iou(labels, tf.reshape(pred_class, [-1]), num_classes=2) n_iou_v = compute_negative_iou(None, n_iou[1]) tf.identity(n_iou_v, 'train_niou') tf.summary.scalar('train/niou', n_iou_v) # add pr curve pr_curve('train/prc', tf.cast(labels, tf.bool), pred_scores, num_thresholds=201) return ce_loss
def _initialize_tensorboard(self): tf.summary.scalar("accuracy", self.accuracy) tf.summary.scalar("cost", self.cost) # for cat in range(self.y.shape[1]): # this is apparently for all classes which I find weird but it works summary_lib.pr_curve( name='pr-curves', predictions=self.pred[:, 1], labels=tf.cast(self.y[:, 1], tf.bool), num_thresholds=self.cf.num_thresholds, display_name="pr-curve (train, test)", description= "Precision versus recall with respective cutoff. Hover over graph for details." ) self.merged = tf.summary.merge_all()
def train_and_evaluate(classifier): # Save a reference to the classifier to run predictions later all_classifiers[classifier.model_dir] = classifier classifier.train(input_fn=train_input_fn, steps=25000) eval_results = classifier.evaluate(input_fn=eval_input_fn) predictions = np.array([p['logistic'][0] for p in classifier.predict(input_fn=eval_input_fn)]) # Reset the graph to be able to reuse name scopes tf.reset_default_graph() # Add a PR summary in addition to the summaries that the classifier writes pr = summary_lib.pr_curve('precision_recall', predictions=predictions, labels=y_test.astype(bool), num_thresholds=21) with tf.Session() as sess: writer = tf.summary.FileWriter(os.path.join(classifier.model_dir, 'eval'), sess.graph) writer.add_summary(sess.run(pr), global_step=0) writer.close()
x_len_train = np.array([min(len(x), sentence_size) for x in x_train_variable]) x_len_test = np.array([min(len(x), sentence_size) for x in x_test_variable]) embedding_matrix = load_glove_embeddings('data/glove.6B.50d.txt', word_index, vocab_size, embedding_size) params = {'embedding_initializer': embedding_matrix} lstm_classifier = tf.estimator.Estimator(model_fn=lstm_model_fn, model_dir=os.path.join(model_dir, 'cnn_pretrained'), params=params) # Save a reference to the classifier to run predictions later lstm_classifier.train(input_fn=train_input_fn(x_train, x_len_train, y_train, x_train_variable), steps=500) eval_results = lstm_classifier.evaluate(input_fn=eval_input_fn(x_test, x_len_test, y_test)) predictions = np.array([p['logistic'][0] for p in lstm_classifier.predict(input_fn=eval_input_fn(x_test, x_len_test, y_test))]) tf.reset_default_graph() pr = summary_lib.pr_curve('precision_recall', predictions=predictions, labels=y_test.astype(bool), num_thresholds=21) with tf.Session() as sess: writer = tf.summary.FileWriter(os.path.join(lstm_classifier.model_dir, 'eval'), sess.graph) writer.add_summary(sess.run(pr), global_step=0) writer.close() def print_predictions(sentences): def text_to_index(sentence): # Remove punctuation characters except for the apostrophe translator = str.maketrans('', '', string.punctuation.replace("'", '')) tokens = sentence.translate(translator).lower().split()