def __init__(self, flags, is_training=True):
self.is_training = is_training
# None = batch_size
self.image_ph = tf.placeholder(tf.float32, shape=(None, flags.feature_size))
self.hard_label_ph = tf.placeholder(tf.float32, shape=(None, flags.num_label))
# None = batch_size * sample_size
self.gen_sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.gen_label_ph = tf.placeholder(tf.float32, shape=(None,))
self.tch_sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.tch_label_ph = tf.placeholder(tf.float32, shape=(None,))
self.dis_scope = dis_scope = 'dis'
model_scope = nets_factory.arg_scopes_map[flags.image_model]
with tf.variable_scope(dis_scope) as scope:
with slim.arg_scope(model_scope(weight_decay=flags.image_weight_decay)):
net = self.image_ph
net = slim.dropout(net, flags.image_keep_prob, is_training=is_training)
net = slim.fully_connected(net, flags.num_label, activation_fn=None)
self.logits = net
self.gen_rewards = self.get_rewards(self.gen_sample_ph)
self.tch_rewards = self.get_rewards(self.tch_sample_ph)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(dis_scope):
continue
print('%-50s added to DIS saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
self.global_step = global_step = tf.Variable(0, trainable=False)
tn_size = utils.get_tn_size(flags.dataset)
learning_rate = flags.dis_learning_rate
self.learning_rate = utils.get_lr(flags, tn_size, global_step, learning_rate, dis_scope)
# pre train
pre_losses = self.get_pre_losses()
pre_losses.extend(self.get_regularization_losses())
print('#pre_losses wt regularization=%d' % (len(pre_losses)))
self.pre_loss = tf.add_n(pre_losses, name='%s_pre_loss' % dis_scope)
pre_optimizer = utils.get_opt(flags, self.learning_rate)
self.pre_update = pre_optimizer.minimize(self.pre_loss, global_step=global_step)
# gan train
gan_losses = self.get_gan_losses(flags)
gan_losses.extend(self.get_regularization_losses())
print('#gan_losses wt regularization=%d' % (len(gan_losses)))
self.gan_loss = tf.add_n(gan_losses, name='%s_gan_loss' % dis_scope)
gan_optimizer = utils.get_opt(flags, self.learning_rate)
self.gan_update = gan_optimizer.minimize(self.gan_loss, global_step=global_step)
from flags import flags
from dis_model import DIS
from gen_model import GEN
from tch_model import TCH
import data_utils
import math
import os
import pickle
import time
import numpy as np
import tensorflow as tf
from os import path
from tensorflow.contrib import slim
tn_size = utils.get_tn_size(flags.dataset)
eval_interval = int(tn_size / flags.batch_size)
print('#tn_size=%d' % (tn_size))
tn_dis = DIS(flags, is_training=True)
tn_gen = GEN(flags, is_training=True)
tn_tch = TCH(flags, is_training=True)
scope = tf.get_variable_scope()
scope.reuse_variables()
vd_dis = DIS(flags, is_training=False)
vd_gen = GEN(flags, is_training=False)
vd_tch = TCH(flags, is_training=False)
for variable in tf.trainable_variables():
num_params = 1
for dim in variable.shape:
def __init__(self, flags, is_training=True):
self.is_training = is_training
# None = batch_size
self.image_ph = tf.placeholder(tf.float32, shape=(None, flags.feature_size))
# None = batch_size * (num_positive + num_negative)
self.sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
# None = batch_size * (num_positive + num_negative)
self.label_ph = tf.placeholder(tf.float32, shape=(None,))
# None = batch_size
self.pre_label_ph = tf.placeholder(tf.float32, shape=(None, config.num_label))
dis_scope = 'discriminator'
model_scope = nets_factory.arg_scopes_map[flags.model_name]
with tf.variable_scope(dis_scope) as scope:
with slim.arg_scope(model_scope(weight_decay=flags.dis_weight_decay)):
net = self.image_ph
net = slim.dropout(net, flags.dropout_keep_prob,
is_training=is_training)
net = slim.fully_connected(net, config.num_label,
activation_fn=None)
self.logits = net
sample_logits = tf.gather_nd(self.logits, self.sample_ph)
# self.rewards = 2 * (tf.sigmoid(sample_logits) - 0.5)
# self.rewards = tf.sigmoid(sample_logits)
reward_logits = self.logits
# reward_logits = 2 * (tf.sigmoid(reward_logits) - 0.5)
# reward_logits -= tf.reduce_mean(reward_logits, 1, keep_dims=True)
# reward_logits -= tf.reduce_mean(reward_logits, 1, keep_dims=True)
# reward_logits = 2 * (tf.sigmoid(reward_logits) - 0.5)
reward_logits = tf.sigmoid(reward_logits)
# reward_logits -= tf.reduce_mean(reward_logits, 1, keep_dims=True)
self.rewards = tf.gather_nd(reward_logits, self.sample_ph)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(dis_scope):
continue
print('%s added to DIS saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
train_data_size = utils.get_tn_size(flags.dataset)
global_step = tf.train.get_global_step()
decay_steps = int(train_data_size / config.train_batch_size * flags.num_epochs_per_decay)
self.learning_rate = tf.train.exponential_decay(flags.init_learning_rate,
global_step, decay_steps, flags.learning_rate_decay_factor,
staircase=True, name='exponential_decay_learning_rate')
# pretrain discriminator
losses = []
losses.append(tf.losses.sigmoid_cross_entropy(self.pre_label_ph, self.logits))
losses.extend(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
self.pre_loss = tf.add_n(losses, name='dis_pre_loss')
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.pre_train_op = optimizer.minimize(self.pre_loss, global_step=global_step)
losses = []
losses.append(tf.losses.sigmoid_cross_entropy(self.label_ph, sample_logits))
regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
losses.extend(regularization_losses)
self.gan_loss = tf.add_n(losses, name='dis_gan_loss')
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.train_op = optimizer.minimize(self.gan_loss, global_step=global_step)
def __init__(self, flags, is_training=True):
self.is_training = is_training
# None = batch_size
self.image_ph = tf.placeholder(tf.float32,
shape=(None, flags.feature_size))
self.hard_label_ph = tf.placeholder(tf.float32,
shape=(None, flags.num_label))
self.soft_logit_ph = tf.placeholder(tf.float32,
shape=(None, flags.num_label))
# None = batch_size * sample_size
self.sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.reward_ph = tf.placeholder(tf.float32, shape=(None, ))
self.gen_scope = gen_scope = 'gen'
model_scope = nets_factory.arg_scopes_map[flags.image_model]
with tf.variable_scope(gen_scope) as scope:
with slim.arg_scope(
model_scope(weight_decay=flags.image_weight_decay)):
net = self.image_ph
net = slim.dropout(net,
flags.image_keep_prob,
is_training=is_training)
net = slim.fully_connected(net,
flags.num_label,
activation_fn=None)
self.logits = net
self.labels = tf.nn.softmax(self.logits)
if not is_training:
return
save_dict, var_list = {}, []
for variable in tf.trainable_variables():
if not variable.name.startswith(gen_scope):
continue
print('%-50s added to GEN saver' % variable.name)
save_dict[variable.name] = variable
var_list.append(variable)
self.saver = tf.train.Saver(save_dict)
self.global_step = global_step = tf.Variable(0, trainable=False)
tn_size = utils.get_tn_size(flags.dataset)
learning_rate = flags.gen_learning_rate
self.learning_rate = utils.get_lr(flags, tn_size, global_step,
learning_rate, gen_scope)
# pre train
pre_losses = self.get_pre_losses()
print('#pre_losses wo regularization=%d' % (len(pre_losses)))
pre_losses.extend(self.get_regularization_losses())
print('#pre_losses wt regularization=%d' % (len(pre_losses)))
self.pre_loss = tf.add_n(pre_losses,
name='%s_pre_loss' % gen_scope)
pre_optimizer = utils.get_opt(flags, self.learning_rate)
self.pre_update = pre_optimizer.minimize(self.pre_loss,
global_step=global_step)
# kd train
kd_losses = self.get_kd_losses(flags)
print('#kd_losses wo regularization=%d' % (len(kd_losses)))
self.kd_loss = tf.add_n(kd_losses, name='%s_kd_loss' % gen_scope)
kd_optimizer = utils.get_opt(flags, self.learning_rate)
self.kd_update = kd_optimizer.minimize(self.kd_loss,
global_step=global_step)
# gan train
gan_losses = self.get_gan_losses()
print('#gan_losses wo regularization=%d' % (len(gan_losses)))
gan_losses.extend(self.get_regularization_losses())
print('#gan_losses wt regularization=%d' % (len(gan_losses)))
self.gan_loss = tf.add_n(gan_losses,
name='%s_gan_loss' % gen_scope)
gan_optimizer = utils.get_opt(flags, self.learning_rate)
self.gan_update = gan_optimizer.minimize(self.gan_loss,
global_step=global_step)
# kdgan train
kdgan_losses = self.get_kdgan_losses(flags)
print('#kdgan_losses wo regularization=%d' % (len(kdgan_losses)))
kdgan_losses.extend(self.get_regularization_losses())
print('#kdgan_losses wt regularization=%d' % (len(kdgan_losses)))
self.kdgan_loss = tf.add_n(kdgan_losses,
name='%s_kdgan_loss' % gen_scope)
kdgan_optimizer = utils.get_opt(flags, self.learning_rate)
# self.kdgan_update = kdgan_optimizer.minimize(self.kdgan_loss, global_step=global_step)
gvs = kdgan_optimizer.compute_gradients(self.kdgan_loss, var_list)
cgvs = [(tf.clip_by_norm(gv[0], config.max_norm), gv[1])
for gv in gvs]
self.kdgan_update = kdgan_optimizer.apply_gradients(
cgvs, global_step=global_step)
def __init__(self, flags, is_training=True):
self.is_training = is_training
# None = batch_size
self.image_ph = tf.placeholder(tf.float32,
shape=(None, flags.feature_size))
self.text_ph = tf.placeholder(tf.int64, shape=(None, None))
self.hard_label_ph = tf.placeholder(tf.float32,
shape=(None, flags.num_label))
# None = batch_size * sample_size
self.sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.reward_ph = tf.placeholder(tf.float32, shape=(None, ))
self.tch_scope = tch_scope = 'tch'
model_scope = nets_factory.arg_scopes_map[flags.image_model]
vocab_size = utils.get_vocab_size(flags.dataset)
with tf.variable_scope(tch_scope) as scope:
with slim.arg_scope(
model_scope(weight_decay=flags.image_weight_decay)):
iembed = self.image_ph
iembed = slim.dropout(iembed,
flags.image_keep_prob,
is_training=is_training)
with slim.arg_scope([slim.fully_connected],
weights_regularizer=slim.l2_regularizer(
flags.text_weight_decay)):
wembed = slim.variable(
'wembed',
shape=[vocab_size, flags.embedding_size],
initializer=tf.random_uniform_initializer(-0.1, 0.1))
tembed = tf.nn.embedding_lookup(wembed, self.text_ph)
tembed = tf.reduce_mean(tembed, axis=-2)
with slim.arg_scope([slim.fully_connected],
weights_regularizer=slim.l2_regularizer(
flags.tch_weight_decay),
biases_initializer=tf.zeros_initializer()):
# cembed = tf.concat([tembed], 1)
cembed = tf.concat([iembed, tembed], 1)
self.logits = slim.fully_connected(cembed,
flags.num_label,
activation_fn=None)
self.labels = tf.nn.softmax(self.logits)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(tch_scope):
continue
print('%-50s added to TCH saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
self.global_step = global_step = tf.Variable(0, trainable=False)
tn_size = utils.get_tn_size(flags.dataset)
learning_rate = flags.tch_learning_rate
self.learning_rate = utils.get_lr(flags, tn_size, global_step,
learning_rate, tch_scope)
# pre train
pre_losses = self.get_pre_losses()
self.pre_loss = tf.add_n(pre_losses,
name='%s_pre_loss' % tch_scope)
pre_losses.extend(self.get_regularization_losses())
print('#pre_losses wt regularization=%d' % (len(pre_losses)))
pre_optimizer = utils.get_opt(flags, self.learning_rate)
self.pre_update = pre_optimizer.minimize(self.pre_loss,
global_step=global_step)
# kdgan train
kdgan_losses = self.get_kdgan_losses(flags)
self.kdgan_loss = tf.add_n(kdgan_losses,
name='%s_kdgan_loss' % tch_scope)
kdgan_optimizer = utils.get_opt(flags, self.learning_rate)
self.kdgan_update = kdgan_optimizer.minimize(
self.kdgan_loss, global_step=global_step)
def __init__(self, flags, is_training=True):
self.is_training = is_training
self.image_ph = tf.placeholder(tf.float32,
shape=(None, flags.feature_size))
self.label_ph = tf.placeholder(tf.float32,
shape=(None, config.num_label))
gen_scope = 'generator'
model_scope = nets_factory.arg_scopes_map[flags.model_name]
with tf.variable_scope(gen_scope) as scope:
with slim.arg_scope(
model_scope(weight_decay=flags.gen_weight_decay)):
net = self.image_ph
net = slim.dropout(net,
flags.dropout_keep_prob,
is_training=is_training)
net = slim.fully_connected(net,
config.num_label,
activation_fn=None)
self.logits = net
self.labels = tf.nn.softmax(self.logits)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(gen_scope):
continue
print('%s added to GEN saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
train_data_size = utils.get_tn_size(flags.dataset)
global_step = tf.train.get_global_step()
decay_steps = int(train_data_size / config.train_batch_size *
flags.num_epochs_per_decay)
self.learning_rate = tf.train.exponential_decay(
flags.init_learning_rate,
global_step,
decay_steps,
flags.learning_rate_decay_factor,
staircase=True,
name='exponential_decay_learning_rate')
# pretrain generator
losses = []
losses.append(
tf.losses.sigmoid_cross_entropy(self.label_ph, self.logits))
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
losses.extend(regularization_losses)
self.pre_loss = tf.add_n(losses, name='pre_loss')
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.train_op = optimizer.minimize(self.pre_loss,
global_step=global_step)
# knowledge distillation
self.hard_label_ph = tf.placeholder(tf.float32,
shape=(None, config.num_label))
self.soft_label_ph = tf.placeholder(tf.float32,
shape=(None, config.num_label))
hard_loss = tf.losses.sigmoid_cross_entropy(self.hard_label_ph,
self.logits)
soft_loss = tf.nn.l2_loss(
tf.nn.softmax(self.logits) -
tf.nn.softmax(self.soft_label_ph / flags.temperature))
kd_loss = (1.0 - flags.beta) * hard_loss + flags.beta * soft_loss
kd_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.kd_train_op = kd_optimizer.minimize(kd_loss,
global_step=global_step)
# generative adversarial network
self.sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.reward_ph = tf.placeholder(tf.float32, shape=(None, ))
sample_logits = tf.gather_nd(self.logits, self.sample_ph)
# gan_loss = -tf.reduce_mean(self.reward_ph * sample_logits)
gan_loss = tf.losses.sigmoid_cross_entropy(self.reward_ph,
sample_logits)
gan_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.gan_train_op = gan_optimizer.minimize(gan_loss,
global_step=global_step)
def __init__(self, flags, is_training=True):
self.is_training = is_training
# None = batch_size
self.image_ph = tf.placeholder(tf.float32,
shape=(None, flags.feature_size))
self.hard_label_ph = tf.placeholder(tf.float32,
shape=(None, flags.num_label))
self.soft_label_ph = tf.placeholder(tf.float32,
shape=(None, flags.num_label))
# None = batch_size * sample_size
self.sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.reward_ph = tf.placeholder(tf.float32, shape=(None, ))
self.gen_scope = gen_scope = 'gen'
model_scope = nets_factory.arg_scopes_map[flags.image_model]
with tf.variable_scope(gen_scope) as scope:
with slim.arg_scope(
model_scope(weight_decay=flags.image_weight_decay)):
net = self.image_ph
net = slim.dropout(net,
flags.image_keep_prob,
is_training=is_training)
net = slim.fully_connected(net,
flags.num_label,
activation_fn=None)
self.logits = net
self.labels = tf.nn.softmax(self.logits)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(gen_scope):
continue
print('%-50s added to GEN saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
self.global_step = global_step = tf.Variable(0, trainable=False)
tn_size = utils.get_tn_size(flags.dataset)
self.learning_rate = utils.get_lr(flags, tn_size, global_step,
flags.gen_learning_rate, gen_scope)
# pre train
pre_losses = []
pre_losses.append(
tf.losses.sigmoid_cross_entropy(self.hard_label_ph, self.logits))
pre_losses.extend(tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES))
self.pre_loss = tf.add_n(pre_losses, name='%s_pre_loss' % gen_scope)
pre_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.pre_update = pre_optimizer.minimize(self.pre_loss,
global_step=global_step)
# kd train
kd_losses = self.get_kd_losses(flags)
self.kd_loss = tf.add_n(kd_losses, name='%s_kd_loss' % gen_scope)
kd_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.kd_update = kd_optimizer.minimize(self.kd_loss,
global_step=global_step)
# gan train
gan_losses = self.get_gan_losses(flags)
gan_losses.extend(tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES))
self.gan_loss = tf.add_n(gan_losses, name='%s_gan_loss' % gen_scope)
gan_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.gan_update = gan_optimizer.minimize(self.gan_loss,
global_step=global_step)
# kdgan train
kdgan_losses = self.get_kd_losses(flags) + self.get_gan_losses(flags)
self.kdgan_loss = tf.add_n(kdgan_losses,
name='%s_kdgan_loss' % gen_scope)
kdgan_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.kdgan_update = kdgan_optimizer.minimize(self.kdgan_loss,
global_step=global_step)
def __init__(self, flags, is_training=True):
self.is_training = is_training
# None = batch_size
self.text_ph = tf.placeholder(tf.int64, shape=(None, None))
self.hard_label_ph = tf.placeholder(tf.float32, shape=(None, flags.num_label))
# None = batch_size * sample_size
self.sample_ph = tf.placeholder(tf.int32, shape=(None, 2))
self.reward_ph = tf.placeholder(tf.float32, shape=(None,))
tch_scope = 'tch'
vocab_size = utils.get_vocab_size(flags.dataset)
# initializer = tf.random_uniform([vocab_size, flags.embedding_size], -0.1, 0.1)
with tf.variable_scope(tch_scope) as scope:
with slim.arg_scope([slim.fully_connected],
weights_regularizer=slim.l2_regularizer(flags.text_weight_decay)):
word_embedding = slim.variable('word_embedding',
shape=[vocab_size, flags.embedding_size],
# regularizer=slim.l2_regularizer(flags.tch_weight_decay),
initializer=tf.random_uniform_initializer(-0.1, 0.1))
# word_embedding = tf.get_variable('word_embedding', initializer=initializer)
text_embedding = tf.nn.embedding_lookup(word_embedding, self.text_ph)
text_embedding = tf.reduce_mean(text_embedding, axis=-2)
self.logits = slim.fully_connected(text_embedding, flags.num_label,
activation_fn=None)
self.labels = tf.nn.softmax(self.logits)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(tch_scope):
continue
print('%-50s added to TCH saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
self.global_step = global_step = tf.Variable(0, trainable=False)
tn_size = utils.get_tn_size(flags.dataset)
self.learning_rate = utils.get_lr(flags,
tn_size,
global_step,
flags.tch_learning_rate,
tch_scope)
# pre train
pre_losses = []
pre_losses.append(tf.losses.sigmoid_cross_entropy(self.hard_label_ph, self.logits))
pre_losses.extend(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
self.pre_loss = tf.add_n(pre_losses, name='%s_pre_loss' % tch_scope)
pre_optimizer = tf.train.AdamOptimizer(self.learning_rate)
self.pre_update = pre_optimizer.minimize(self.pre_loss, global_step=global_step)
# kdgan train
sample_logits = tf.gather_nd(self.logits, self.sample_ph)
kdgan_losses = [tf.losses.sigmoid_cross_entropy(self.reward_ph, sample_logits)]
self.kdgan_loss = tf.add_n(kdgan_losses, name='%s_kdgan_loss' % tch_scope)
kdgan_optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.kdgan_update = kdgan_optimizer.minimize(self.kdgan_loss, global_step=global_step)
def __init__(self, flags, is_training=True):
self.is_training = is_training
self.text_ph = tf.placeholder(tf.int64, shape=(None, None))
self.label_ph = tf.placeholder(tf.float32,
shape=(None, config.num_label))
tch_scope = 'teacher'
vocab_size = utils.get_vocab_size(flags.dataset)
# initializer = tf.random_uniform([vocab_size, flags.embedding_size], -0.1, 0.1)
with tf.variable_scope(tch_scope) as scope:
with slim.arg_scope([slim.fully_connected],
weights_regularizer=slim.l2_regularizer(
flags.tch_weight_decay)):
word_embedding = slim.variable(
'word_embedding',
shape=[vocab_size, flags.embedding_size],
# regularizer=slim.l2_regularizer(flags.tch_weight_decay),
initializer=tf.random_uniform_initializer(-0.1, 0.1))
# word_embedding = tf.get_variable('word_embedding', initializer=initializer)
text_embedding = tf.nn.embedding_lookup(
word_embedding, self.text_ph)
text_embedding = tf.reduce_mean(text_embedding, axis=-2)
self.logits = slim.fully_connected(text_embedding,
config.num_label,
activation_fn=None)
self.labels = tf.nn.softmax(self.logits)
if not is_training:
return
save_dict = {}
for variable in tf.trainable_variables():
if not variable.name.startswith(tch_scope):
continue
print('%s added to TCH saver' % variable.name)
save_dict[variable.name] = variable
self.saver = tf.train.Saver(save_dict)
train_data_size = utils.get_tn_size(flags.dataset)
global_step = tf.train.get_global_step()
decay_steps = int(train_data_size / config.train_batch_size *
flags.num_epochs_per_decay)
self.learning_rate = tf.train.exponential_decay(
flags.init_learning_rate,
global_step,
decay_steps,
flags.learning_rate_decay_factor,
staircase=True,
name='exponential_decay_learning_rate')
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=self.label_ph,
logits=self.logits))
losses = [loss]
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
losses.extend(regularization_losses)
total_loss = tf.add_n(losses, name='total_loss')
optimizer = tf.train.AdamOptimizer(self.learning_rate)
self.train_op = optimizer.minimize(total_loss, global_step=global_step)
tf.summary.scalar('total_loss', total_loss)
self.summary_op = tf.summary.merge_all()
