def four_layer_binary_classifier(features, labels, mode, params):
"""
Estimator wrapper function for four-layer cnn performing binary classification
Details (e.g. neurons, activation funcs, etc) controlled by 'params'
Args:
features ():
labels ():
mode ():
params ():
Returns:
"""
predictions, loss = four_layer_cnn(features, labels, mode, params)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = params['optimizer'](learning_rate=params['learning_rate'])
train_op = optimizer.minimize(loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
tensorboard_summary.pr_curve_streaming_op(
name='spirals',
labels=labels,
predictions=predictions['probabilities'][:, 1],
)
# Add evaluation metrics (for EVAL mode)
eval_metric_ops = get_eval_metric_ops(labels, predictions)
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
eval_metric_ops=eval_metric_ops)
def test_compute_precision_recall(self):
num_gt = 10
scores = np.array([0.4, 0.3, 0.6, 0.2, 0.7, 0.1], dtype=float)
labels = np.array([0, 1, 1, 0, 0, 1], dtype=bool)
labels_float_type = np.array([0, 1, 1, 0, 0, 1], dtype=float)
accumulated_tp_count = np.array([0, 1, 1, 2, 2, 3], dtype=float)
expected_precision = accumulated_tp_count / np.array(
[1, 2, 3, 4, 5, 6])
expected_recall = accumulated_tp_count / num_gt
precision, recall = metrics.compute_precision_recall(
scores, labels, num_gt)
precision_float_type, recall_float_type = metrics.compute_precision_recall(
scores, labels_float_type, num_gt)
# Add a module to export PR curve Data into an event file for showing in Tensorboard
#labels = tf.constant([False, True, True, False, True], dtype=tf.bool)
#predictions = tf.random_uniform(labels.get_shape(), maxval=1.0)
_, update_op = summary_lib.pr_curve_streaming_op(name='PR_Curve',
predictions=scores,
labels=labels,
num_thresholds=10)
merged_summary = tf.summary.merge_all()
with tf.Session() as sess:
writer = tf.summary.FileWriter('/home/ai-lab/frcnn/tmp/logdir',
sess.graph)
sess.run(tf.local_variables_initializer())
for step in range(43):
sess.run([update_op])
if step % 6 == 0:
writer.add_summary(sess.run(merged_summary),
global_step=step)
# These above lines was added by Huy Vu!
self.assertAllClose(precision, expected_precision)
self.assertAllClose(recall, expected_recall)
self.assertAllClose(precision_float_type, expected_precision)
self.assertAllClose(recall_float_type, expected_recall)
precision, prec_op = tf.metrics.precision(labels=y, predictions=predictions, updates_collections=tf.GraphKeys.UPDATE_OPS, name="precision")
f1_score = 2 * ( (precision * recall) / (precision + recall))
#auc, auc_op = tf.metrics.auc(labels=y, predictions=probabilities[:,1], num_thresholds=50, name="auc_1", updates_collections=tf.GraphKeys.UPDATE_OPS)
#tf.summary.scalar('auc_', auc, collections=["summaries"])
# Create summary hooks
tf.summary.scalar('accuracy', accuracy, collections=["summaries"])
tf.summary.scalar('recall_1', recall, collections=["summaries"])
tf.summary.scalar('cross_entropy', mean_ce, collections=["summaries"])
tf.summary.scalar('learning_rate', learning_rate, collections=["summaries"])
_, update_op = summary_lib.pr_curve_streaming_op(name='pr_curve',
predictions=probabilities[:,1],
labels=y,
updates_collections=tf.GraphKeys.UPDATE_OPS,
#metrics_collections=["summaries"],
num_thresholds=20)
if num_classes == 2:
tf.summary.scalar('precision_1', precision, collections=["summaries"])
tf.summary.scalar('f1_score', f1_score, collections=["summaries"])
# add this so that the batch norm gets run
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# Merge all the summaries
merged = tf.summary.merge_all()
kernel_summaries = tf.summary.merge_all("kernels")
print("Graph created...")
# ## Train
def train():
vocab_id, vocab_info, _ = utils.load_vocab_details(_VOCAB_PATH)
train_graph = tf.Graph()
train_session = tf.Session(config=CONF, graph=train_graph)
with train_graph.as_default(), tf.container("train"):
embedding = utils._create_pretrained_emb_from_txt(
_VOCAB_PATH, _EMBEDDING_PATH)
print type(embedding)
model = qa.CQA(vocab_id, embedding, qa.Mode.train)
model.create_or_load_model(train_session, qa.FLAGS.model_dir)
dev_graph = tf.Graph()
dev_session = tf.Session(graph=dev_graph)
with dev_graph.as_default(), tf.container("dev"):
predict_result_ph = tf.placeholder(dtype=tf.float32)
ground_ph = tf.placeholder(dtype=tf.bool)
embedding = utils._create_pretrained_emb_from_txt(
_VOCAB_PATH, _EMBEDDING_PATH)
dev_model = qa.CQA(vocab_id, embedding, qa.Mode.test)
k = 1
_, update_op = summary_lib.pr_curve_streaming_op(
tag="pr_at_1", predictions=predict_result_ph, labels=ground_ph)
averaged_precision_score_ph = tf.placeholder(
dtype=tf.float32, name="averaged_precision_score")
area_under_prcurve_ph = tf.placeholder(dtype=tf.float32,
name="area_under_prcurve")
tf.summary.scalar("averaged_precision_score",
averaged_precision_score_ph)
tf.summary.scalar("area_under_prcurve", area_under_prcurve_ph)
pr_summary = tf.summary.merge_all()
dev_session.run(tf.local_variables_initializer())
summary_writer = tf.summary.FileWriter(qa.FLAGS.tb_dir,
train_session.graph)
for epoch in xrange(qa.FLAGS.epoch):
'''for q, a, wa in data_utils.neg_sampled_data_generator(os.path.join(qa.FLAGS.data_dir, "cqa/train_q.neg_sampled"),
qa.FLAGS.batch_size,
qa.FLAGS.neg_sample_num,
qa.FLAGS.sentence_len):
'''
for q, a, wa in data_utils.neg_sampled_data_generator(
_TRAIN_PATH, qa.FLAGS.batch_size, qa.FLAGS.neg_sample_num,
qa.FLAGS.sentence_len):
q = np.reshape(
np.asarray(map(lambda x: [x] * qa.FLAGS.neg_sample_num, q)), [
qa.FLAGS.batch_size * qa.FLAGS.neg_sample_num,
qa.FLAGS.sentence_len
])
a = np.reshape(
np.asarray(map(lambda x: [x] * qa.FLAGS.neg_sample_num, a)), [
qa.FLAGS.batch_size * qa.FLAGS.neg_sample_num,
qa.FLAGS.sentence_len
])
wa = np.reshape(np.asarray(wa), [
qa.FLAGS.batch_size * qa.FLAGS.neg_sample_num,
qa.FLAGS.sentence_len
])
cos_q_a, loss, update_step, summaries = model.train(
train_session, q, a, wa)
step = model.global_step.eval(train_session)
summary_writer.add_summary(summaries, step)
print "epoch: %s, step: %s, loss: %s" % (epoch, step, str(loss))
if model.global_step.eval(train_session) % 100 == 0:
print "save model to: %s/ckpt-%s" % (
qa.FLAGS.model_dir, model.global_step.eval(train_session))
model.saver.save(train_session,
"%s/ckpt-" % qa.FLAGS.model_dir,
global_step=model.global_step)
if model.global_step.eval(train_session) % 100 == 0:
# do eval
if qa.FLAGS.with_margin_loss:
truncated_predicted_results, truncated_grounded_results, p, r, threshold, averaged_precision_score, area_under_prcurve = evaluate.pr_at_k(
k, dev_model, dev_session, qa.FLAGS.model_dir,
evaluate._DEV_Q_PATH, qa.FLAGS.eval_dir)
summary_writer.add_summary(dev_session.run(
pr_summary,
feed_dict={
predict_result_ph:
truncated_predicted_results,
ground_ph:
map(lambda x: True if int(x) else False,
truncated_grounded_results),
averaged_precision_score_ph:
averaged_precision_score,
area_under_prcurve_ph:
area_under_prcurve
}),
global_step=step)
if qa.FLAGS.with_softmax:
truncated_predicted_results, truncated_grounded_results, p, r, threshold, averaged_precision_score, area_under_prcurve = evaluate.pr_at_k(
k,
dev_model,
dev_session,
qa.FLAGS.model_dir,
evaluate._DEV_Q_PATH,
qa.FLAGS.eval_dir,
predictMode=PredictMode.softmax)
summary_writer.add_summary(dev_session.run(
pr_summary,
feed_dict={
predict_result_ph:
truncated_predicted_results,
ground_ph:
map(lambda x: True if int(x) else False,
truncated_grounded_results),
averaged_precision_score_ph:
averaged_precision_score,
area_under_prcurve_ph:
area_under_prcurve
}),
global_step=step)
f1_score = 2 * ((precision * recall) / (precision + recall))
# add this so that the batch norm gets run
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# Create summary hooks
tf.summary.scalar('accuracy', accuracy, collections=["summaries"])
tf.summary.scalar('recall_1', recall, collections=["summaries"])
tf.summary.scalar('cross_entropy', mean_ce, collections=["summaries"])
tf.summary.scalar('loss', loss, collections=["summaries"])
tf.summary.scalar('learning_rate',
learning_rate,
collections=["summaries"])
_, update_op = summary_lib.pr_curve_streaming_op(name='pr_curve',
predictions=predictions,
labels=y,
num_thresholds=10)
if num_classes == 2:
tf.summary.scalar('precision_1', precision, collections=["summaries"])
tf.summary.scalar('f1_score', f1_score, collections=["summaries"])
# Merge all the summaries and write them out to /tmp/mnist_logs (by default)
merged = tf.summary.merge_all()
#pr_curve = tf.summary.merge_all("pr")
print("Graph created...")
# ## Train
## CONFIGURE OPTIONS
if os.path.exists(os.path.join("model", model_name + '.ckpt.index')):
init = False
else:
recall, rec_op = tf.metrics.recall(
labels=y,
predictions=predictions,
updates_collections=tf.GraphKeys.UPDATE_OPS,
name="recall")
precision, prec_op = tf.metrics.precision(
labels=y,
predictions=predictions,
updates_collections=tf.GraphKeys.UPDATE_OPS,
name="precision")
f1_score = 2 * ((precision * recall) / (precision + recall))
_, update_op = summary_lib.pr_curve_streaming_op(
name='pr_curve',
predictions=(1 - probabilities[:, 0]),
labels=y,
updates_collections=tf.GraphKeys.UPDATE_OPS,
num_thresholds=20)
tf.summary.scalar('recall_1', recall, collections=["summaries"])
tf.summary.scalar('precision_1', precision, collections=["summaries"])
tf.summary.scalar('f1_score', f1_score, collections=["summaries"])
# Create summary hooks
tf.summary.scalar('accuracy', accuracy, collections=["summaries"])
tf.summary.scalar('cross_entropy', mean_ce, collections=["summaries"])
tf.summary.scalar('learning_rate',
learning_rate,
collections=["summaries"])
# add this so that the batch norm gets run
def model_fn(self, features, labels, mode):
tf.summary.image("original", features["A/original"])
# build generator
with tf.variable_scope("UNet"):
self._generate(features["A/image"])
# prediction dict
self._predictions = {
"generated_soft": self._gen_image,
"generated_segmentation": self._bin_image
}
# build loss
if mode != tf.estimator.ModeKeys.PREDICT:
# normal unet loss
with tf.name_scope("unet_loss"):
classify_loss = self._classification_loss(features["B/image"])
if mode == tf.estimator.ModeKeys.EVAL:
self._evals["pr_curve"] = summary_lib.pr_curve_streaming_op(
name='pr_curve',
predictions=self._gen_image,
labels=features["B/image"],
num_thresholds=11)
total_los = classify_loss
tf.summary.image(
"unet_loss/result",
tf.concat(
[self._gen_image, self._gen_image, features["B/image"]],
axis=3))
else:
total_los = None
# connected components
cct = tf.contrib.image.connected_components(
tf.less(features["B/image"][:, :, :, 0], 0.5))
ccr = tf.contrib.image.connected_components(1.0 -
self._bin_image[:, :, :,
0])
tf.summary.image("connected_components_target",
tf.cast(cct, tf.float32)[:, :, :, None])
tf.summary.image("connected_components_result",
tf.cast(ccr, tf.float32)[:, :, :, None])
self._predictions["connected_components"] = ccr
# GAN loss
if self.model.config.discrimination_loss > 0.0:
disc_loss, gan_loss = self._gan_loss(
features, mode != tf.estimator.ModeKeys.PREDICT)
if mode != tf.estimator.ModeKeys.PREDICT:
total_los = classify_loss + gan_loss
iou = iou_from_intersection_op(cct, ccr)
tf.summary.scalar("iou", tf.reduce_mean(iou))
if mode == tf.estimator.ModeKeys.EVAL:
self._evals["iou"] = tf.metrics.mean(tf.reduce_mean(iou))
if mode == tf.estimator.ModeKeys.TRAIN:
# boundaries = [2000, 5000]
# values = [1e-3, 5e-4, 1e-4]
learning_rate = 1e-3 # tf.train.piecewise_constant(tf.train.get_or_create_global_step(), boundaries, values)
g_optimizer = tf.train.AdamOptimizer(learning_rate)
train_step = g_optimizer.minimize(
total_los,
global_step=tf.train.get_or_create_global_step(),
var_list=tf.trainable_variables("UNet"))
# discriminator update
if self.model.config.discrimination_loss > 0.0:
# only start training the discriminator after we have done some UNet training
lr = tf.train.piecewise_constant(
tf.train.get_or_create_global_step(), [250],
[0.0, self.model.config.disc_lr])
if self.model.config.disc_opt.upper() == "ADAM":
d_optimizer = tf.train.AdamOptimizer(lr)
elif self.model.config.disc_opt.upper() == "SGD":
d_optimizer = tf.train.MomentumOptimizer(lr, momentum=0.9)
else:
raise NotImplementedError()
g_step = train_step
d_step = d_optimizer.minimize(
disc_loss,
global_step=None,
var_list=tf.trainable_variables("Discriminator"))
disc_up_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
"Discriminator")
train_step = tf.group((g_step, d_step) + tuple(disc_up_ops))
else:
train_step = None
return tf.estimator.EstimatorSpec(mode=mode,
predictions=self._predictions,
loss=total_los,
eval_metric_ops=self._evals,
train_op=train_step)
