def four_layer_binary_classifier(features, labels, mode, params): """ Estimator wrapper function for four-layer cnn performing binary classification Details (e.g. neurons, activation funcs, etc) controlled by 'params' Args: features (): labels (): mode (): params (): Returns: """ predictions, loss = four_layer_cnn(features, labels, mode, params) if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) if mode == tf.estimator.ModeKeys.TRAIN: optimizer = params['optimizer'](learning_rate=params['learning_rate']) train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) tensorboard_summary.pr_curve_streaming_op( name='spirals', labels=labels, predictions=predictions['probabilities'][:, 1], ) # Add evaluation metrics (for EVAL mode) eval_metric_ops = get_eval_metric_ops(labels, predictions) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def test_compute_precision_recall(self): num_gt = 10 scores = np.array([0.4, 0.3, 0.6, 0.2, 0.7, 0.1], dtype=float) labels = np.array([0, 1, 1, 0, 0, 1], dtype=bool) labels_float_type = np.array([0, 1, 1, 0, 0, 1], dtype=float) accumulated_tp_count = np.array([0, 1, 1, 2, 2, 3], dtype=float) expected_precision = accumulated_tp_count / np.array( [1, 2, 3, 4, 5, 6]) expected_recall = accumulated_tp_count / num_gt precision, recall = metrics.compute_precision_recall( scores, labels, num_gt) precision_float_type, recall_float_type = metrics.compute_precision_recall( scores, labels_float_type, num_gt) # Add a module to export PR curve Data into an event file for showing in Tensorboard #labels = tf.constant([False, True, True, False, True], dtype=tf.bool) #predictions = tf.random_uniform(labels.get_shape(), maxval=1.0) _, update_op = summary_lib.pr_curve_streaming_op(name='PR_Curve', predictions=scores, labels=labels, num_thresholds=10) merged_summary = tf.summary.merge_all() with tf.Session() as sess: writer = tf.summary.FileWriter('/home/ai-lab/frcnn/tmp/logdir', sess.graph) sess.run(tf.local_variables_initializer()) for step in range(43): sess.run([update_op]) if step % 6 == 0: writer.add_summary(sess.run(merged_summary), global_step=step) # These above lines was added by Huy Vu! self.assertAllClose(precision, expected_precision) self.assertAllClose(recall, expected_recall) self.assertAllClose(precision_float_type, expected_precision) self.assertAllClose(recall_float_type, expected_recall)
precision, prec_op = tf.metrics.precision(labels=y, predictions=predictions, updates_collections=tf.GraphKeys.UPDATE_OPS, name="precision") f1_score = 2 * ( (precision * recall) / (precision + recall)) #auc, auc_op = tf.metrics.auc(labels=y, predictions=probabilities[:,1], num_thresholds=50, name="auc_1", updates_collections=tf.GraphKeys.UPDATE_OPS) #tf.summary.scalar('auc_', auc, collections=["summaries"]) # Create summary hooks tf.summary.scalar('accuracy', accuracy, collections=["summaries"]) tf.summary.scalar('recall_1', recall, collections=["summaries"]) tf.summary.scalar('cross_entropy', mean_ce, collections=["summaries"]) tf.summary.scalar('learning_rate', learning_rate, collections=["summaries"]) _, update_op = summary_lib.pr_curve_streaming_op(name='pr_curve', predictions=probabilities[:,1], labels=y, updates_collections=tf.GraphKeys.UPDATE_OPS, #metrics_collections=["summaries"], num_thresholds=20) if num_classes == 2: tf.summary.scalar('precision_1', precision, collections=["summaries"]) tf.summary.scalar('f1_score', f1_score, collections=["summaries"]) # add this so that the batch norm gets run extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Merge all the summaries merged = tf.summary.merge_all() kernel_summaries = tf.summary.merge_all("kernels") print("Graph created...") # ## Train
def train(): vocab_id, vocab_info, _ = utils.load_vocab_details(_VOCAB_PATH) train_graph = tf.Graph() train_session = tf.Session(config=CONF, graph=train_graph) with train_graph.as_default(), tf.container("train"): embedding = utils._create_pretrained_emb_from_txt( _VOCAB_PATH, _EMBEDDING_PATH) print type(embedding) model = qa.CQA(vocab_id, embedding, qa.Mode.train) model.create_or_load_model(train_session, qa.FLAGS.model_dir) dev_graph = tf.Graph() dev_session = tf.Session(graph=dev_graph) with dev_graph.as_default(), tf.container("dev"): predict_result_ph = tf.placeholder(dtype=tf.float32) ground_ph = tf.placeholder(dtype=tf.bool) embedding = utils._create_pretrained_emb_from_txt( _VOCAB_PATH, _EMBEDDING_PATH) dev_model = qa.CQA(vocab_id, embedding, qa.Mode.test) k = 1 _, update_op = summary_lib.pr_curve_streaming_op( tag="pr_at_1", predictions=predict_result_ph, labels=ground_ph) averaged_precision_score_ph = tf.placeholder( dtype=tf.float32, name="averaged_precision_score") area_under_prcurve_ph = tf.placeholder(dtype=tf.float32, name="area_under_prcurve") tf.summary.scalar("averaged_precision_score", averaged_precision_score_ph) tf.summary.scalar("area_under_prcurve", area_under_prcurve_ph) pr_summary = tf.summary.merge_all() dev_session.run(tf.local_variables_initializer()) summary_writer = tf.summary.FileWriter(qa.FLAGS.tb_dir, train_session.graph) for epoch in xrange(qa.FLAGS.epoch): '''for q, a, wa in data_utils.neg_sampled_data_generator(os.path.join(qa.FLAGS.data_dir, "cqa/train_q.neg_sampled"), qa.FLAGS.batch_size, qa.FLAGS.neg_sample_num, qa.FLAGS.sentence_len): ''' for q, a, wa in data_utils.neg_sampled_data_generator( _TRAIN_PATH, qa.FLAGS.batch_size, qa.FLAGS.neg_sample_num, qa.FLAGS.sentence_len): q = np.reshape( np.asarray(map(lambda x: [x] * qa.FLAGS.neg_sample_num, q)), [ qa.FLAGS.batch_size * qa.FLAGS.neg_sample_num, qa.FLAGS.sentence_len ]) a = np.reshape( np.asarray(map(lambda x: [x] * qa.FLAGS.neg_sample_num, a)), [ qa.FLAGS.batch_size * qa.FLAGS.neg_sample_num, qa.FLAGS.sentence_len ]) wa = np.reshape(np.asarray(wa), [ qa.FLAGS.batch_size * qa.FLAGS.neg_sample_num, qa.FLAGS.sentence_len ]) cos_q_a, loss, update_step, summaries = model.train( train_session, q, a, wa) step = model.global_step.eval(train_session) summary_writer.add_summary(summaries, step) print "epoch: %s, step: %s, loss: %s" % (epoch, step, str(loss)) if model.global_step.eval(train_session) % 100 == 0: print "save model to: %s/ckpt-%s" % ( qa.FLAGS.model_dir, model.global_step.eval(train_session)) model.saver.save(train_session, "%s/ckpt-" % qa.FLAGS.model_dir, global_step=model.global_step) if model.global_step.eval(train_session) % 100 == 0: # do eval if qa.FLAGS.with_margin_loss: truncated_predicted_results, truncated_grounded_results, p, r, threshold, averaged_precision_score, area_under_prcurve = evaluate.pr_at_k( k, dev_model, dev_session, qa.FLAGS.model_dir, evaluate._DEV_Q_PATH, qa.FLAGS.eval_dir) summary_writer.add_summary(dev_session.run( pr_summary, feed_dict={ predict_result_ph: truncated_predicted_results, ground_ph: map(lambda x: True if int(x) else False, truncated_grounded_results), averaged_precision_score_ph: averaged_precision_score, area_under_prcurve_ph: area_under_prcurve }), global_step=step) if qa.FLAGS.with_softmax: truncated_predicted_results, truncated_grounded_results, p, r, threshold, averaged_precision_score, area_under_prcurve = evaluate.pr_at_k( k, dev_model, dev_session, qa.FLAGS.model_dir, evaluate._DEV_Q_PATH, qa.FLAGS.eval_dir, predictMode=PredictMode.softmax) summary_writer.add_summary(dev_session.run( pr_summary, feed_dict={ predict_result_ph: truncated_predicted_results, ground_ph: map(lambda x: True if int(x) else False, truncated_grounded_results), averaged_precision_score_ph: averaged_precision_score, area_under_prcurve_ph: area_under_prcurve }), global_step=step)
f1_score = 2 * ((precision * recall) / (precision + recall)) # add this so that the batch norm gets run extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # Create summary hooks tf.summary.scalar('accuracy', accuracy, collections=["summaries"]) tf.summary.scalar('recall_1', recall, collections=["summaries"]) tf.summary.scalar('cross_entropy', mean_ce, collections=["summaries"]) tf.summary.scalar('loss', loss, collections=["summaries"]) tf.summary.scalar('learning_rate', learning_rate, collections=["summaries"]) _, update_op = summary_lib.pr_curve_streaming_op(name='pr_curve', predictions=predictions, labels=y, num_thresholds=10) if num_classes == 2: tf.summary.scalar('precision_1', precision, collections=["summaries"]) tf.summary.scalar('f1_score', f1_score, collections=["summaries"]) # Merge all the summaries and write them out to /tmp/mnist_logs (by default) merged = tf.summary.merge_all() #pr_curve = tf.summary.merge_all("pr") print("Graph created...") # ## Train ## CONFIGURE OPTIONS if os.path.exists(os.path.join("model", model_name + '.ckpt.index')): init = False
else: recall, rec_op = tf.metrics.recall( labels=y, predictions=predictions, updates_collections=tf.GraphKeys.UPDATE_OPS, name="recall") precision, prec_op = tf.metrics.precision( labels=y, predictions=predictions, updates_collections=tf.GraphKeys.UPDATE_OPS, name="precision") f1_score = 2 * ((precision * recall) / (precision + recall)) _, update_op = summary_lib.pr_curve_streaming_op( name='pr_curve', predictions=(1 - probabilities[:, 0]), labels=y, updates_collections=tf.GraphKeys.UPDATE_OPS, num_thresholds=20) tf.summary.scalar('recall_1', recall, collections=["summaries"]) tf.summary.scalar('precision_1', precision, collections=["summaries"]) tf.summary.scalar('f1_score', f1_score, collections=["summaries"]) # Create summary hooks tf.summary.scalar('accuracy', accuracy, collections=["summaries"]) tf.summary.scalar('cross_entropy', mean_ce, collections=["summaries"]) tf.summary.scalar('learning_rate', learning_rate, collections=["summaries"]) # add this so that the batch norm gets run
def model_fn(self, features, labels, mode): tf.summary.image("original", features["A/original"]) # build generator with tf.variable_scope("UNet"): self._generate(features["A/image"]) # prediction dict self._predictions = { "generated_soft": self._gen_image, "generated_segmentation": self._bin_image } # build loss if mode != tf.estimator.ModeKeys.PREDICT: # normal unet loss with tf.name_scope("unet_loss"): classify_loss = self._classification_loss(features["B/image"]) if mode == tf.estimator.ModeKeys.EVAL: self._evals["pr_curve"] = summary_lib.pr_curve_streaming_op( name='pr_curve', predictions=self._gen_image, labels=features["B/image"], num_thresholds=11) total_los = classify_loss tf.summary.image( "unet_loss/result", tf.concat( [self._gen_image, self._gen_image, features["B/image"]], axis=3)) else: total_los = None # connected components cct = tf.contrib.image.connected_components( tf.less(features["B/image"][:, :, :, 0], 0.5)) ccr = tf.contrib.image.connected_components(1.0 - self._bin_image[:, :, :, 0]) tf.summary.image("connected_components_target", tf.cast(cct, tf.float32)[:, :, :, None]) tf.summary.image("connected_components_result", tf.cast(ccr, tf.float32)[:, :, :, None]) self._predictions["connected_components"] = ccr # GAN loss if self.model.config.discrimination_loss > 0.0: disc_loss, gan_loss = self._gan_loss( features, mode != tf.estimator.ModeKeys.PREDICT) if mode != tf.estimator.ModeKeys.PREDICT: total_los = classify_loss + gan_loss iou = iou_from_intersection_op(cct, ccr) tf.summary.scalar("iou", tf.reduce_mean(iou)) if mode == tf.estimator.ModeKeys.EVAL: self._evals["iou"] = tf.metrics.mean(tf.reduce_mean(iou)) if mode == tf.estimator.ModeKeys.TRAIN: # boundaries = [2000, 5000] # values = [1e-3, 5e-4, 1e-4] learning_rate = 1e-3 # tf.train.piecewise_constant(tf.train.get_or_create_global_step(), boundaries, values) g_optimizer = tf.train.AdamOptimizer(learning_rate) train_step = g_optimizer.minimize( total_los, global_step=tf.train.get_or_create_global_step(), var_list=tf.trainable_variables("UNet")) # discriminator update if self.model.config.discrimination_loss > 0.0: # only start training the discriminator after we have done some UNet training lr = tf.train.piecewise_constant( tf.train.get_or_create_global_step(), [250], [0.0, self.model.config.disc_lr]) if self.model.config.disc_opt.upper() == "ADAM": d_optimizer = tf.train.AdamOptimizer(lr) elif self.model.config.disc_opt.upper() == "SGD": d_optimizer = tf.train.MomentumOptimizer(lr, momentum=0.9) else: raise NotImplementedError() g_step = train_step d_step = d_optimizer.minimize( disc_loss, global_step=None, var_list=tf.trainable_variables("Discriminator")) disc_up_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, "Discriminator") train_step = tf.group((g_step, d_step) + tuple(disc_up_ops)) else: train_step = None return tf.estimator.EstimatorSpec(mode=mode, predictions=self._predictions, loss=total_los, eval_metric_ops=self._evals, train_op=train_step)