def create_Q_network(self):
# input layer
self.state_input = tf.placeholder("float", [None, self.state_dim])
self.ISWeights = tf.placeholder(tf.float32, [None, 1])
# network weights
with tf.variable_scope('current_net'):
W1 = self.weight_variable([self.state_dim,20])
b1 = self.bias_variable([20])
W2 = self.weight_variable([20,self.action_dim])
b2 = self.bias_variable([self.action_dim])
# hidden layers
h_layer = tf.nn.relu(tf.matmul(self.state_input,W1) + b1)
# Q Value layer
self.Q_value = tf.matmul(h_layer,W2) + b2
with tf.variable_scope('target_net'):
W1t = self.weight_variable([self.state_dim,20])
b1t = self.bias_variable([20])
W2t = self.weight_variable([20,self.action_dim])
b2t = self.bias_variable([self.action_dim])
# hidden layers
h_layer_t = tf.nn.relu(tf.matmul(self.state_input,W1t) + b1t)
# Q Value layer
self.target_Q_value = tf.matmul(h_layer,W2t) + b2t
t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='target_net')
e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='current_net')
with tf.variable_scope('soft_replacement'):
self.target_replace_op = [tf.assign(t, e) for t, e in zip(t_params, e_params)]
def get_test_collection(self):
ret = dict()
ret['cls_score'] = tf.get_collection('cls_score')[0]
ret['cls_prob'] = tf.get_collection('cls_prob')[0]
ret['bbox_pred'] = tf.get_collection('bbox_pred')[0]
ret['rois'] = tf.get_collection('rois')[0]
return ret
def testVariableCollections(self):
with self.test_session():
a = tf.contrib.framework.variable('a', [], collections=['A', 'C'])
b = tf.contrib.framework.variable('b', [], collections=['B', 'C'])
self.assertEquals(a, tf.get_collection('A')[0])
self.assertEquals(b, tf.get_collection('B')[0])
self.assertListEqual([a, b], tf.get_collection('C'))
def build_model(self):
self.x = tf.placeholder(tf.float32, [self.reader.vocab_size], name="input")
self.x_idx = tf.placeholder(tf.int32, [None], name='x_idx') # mask paddings
self.build_encoder()
self.build_generator()
self.objective = self.kl +self.recons_loss
# optimizer for alternative update
optimizer1 = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
optimizer2 = tf.train.AdamOptimizer(learning_rate=0.1)
fullvars = tf.GraphKeys.TRAINABLE_VARIABLES
print 'fullvars:',fullvars
enc_vars = tf.get_collection(fullvars,scope='encoder')
print enc_vars
dec_vars = tf.get_collection(fullvars,scope='generator')
print dec_vars
self.lossL2_enc = tf.add_n([ tf.nn.l2_loss(v) for v in enc_vars if 'bias' not in v.name]) * 0.0001
self.lossL2_dec = tf.add_n([ tf.nn.l2_loss(v) for v in dec_vars if 'bias' not in v.name])
print 'lossL2_enc:',self.lossL2_enc
print 'lossL2_dec:',self.lossL2_dec
enc_grads = tf.gradients(self.kl+self.lossL2_enc, enc_vars)
dec_grads = tf.gradients(self.recons_loss+self.lossL2_dec, dec_vars)
self.optim_enc = optimizer1.apply_gradients(zip(enc_grads, enc_vars))
self.optim_dec = optimizer2.apply_gradients(zip(dec_grads, dec_vars))
def optimize_test(self):
# Test time optimizer to compare log-likelihood score of ZINB-WaVE
update_ops_test = tf.get_collection(tf.GraphKeys.UPDATE_OPS, "variational")
test_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "variational")
optimizer_test = tf.train.AdamOptimizer(learning_rate=0.001, epsilon=0.1)
with tf.control_dependencies(update_ops_test):
self.test_step = optimizer_test.minimize(self.loss, var_list=test_vars)
def testVarScopeRegularizer(self):
with self.test_session() as sess:
init = tf.constant_initializer(0.3)
def regularizer1(v):
return tf.reduce_mean(v) + 0.1
def regularizer2(v):
return tf.reduce_mean(v) + 0.2
with tf.variable_scope("tower", regularizer=regularizer1) as tower:
with tf.variable_scope("foo", initializer=init):
v = tf.get_variable("v", [])
sess.run(tf.initialize_variables([v]))
losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
self.assertEqual(1, len(losses))
self.assertAllClose(losses[0].eval(), 0.4)
with tf.variable_scope(tower, initializer=init) as vs:
u = tf.get_variable("u", [])
vs.set_regularizer(regularizer2)
w = tf.get_variable("w", [])
# Next 3 variable not regularized to test disabling regularization.
x = tf.get_variable("x", [], regularizer=tf.no_regularizer)
with tf.variable_scope("baz", regularizer=tf.no_regularizer):
y = tf.get_variable("y", [])
vs.set_regularizer(tf.no_regularizer)
z = tf.get_variable("z", [])
# Check results.
losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
self.assertEqual(3, len(losses))
sess.run(tf.initialize_variables([u, w, x, y, z]))
self.assertAllClose(losses[0].eval(), 0.4)
self.assertAllClose(losses[1].eval(), 0.4)
self.assertAllClose(losses[2].eval(), 0.5)
with tf.variable_scope("foo", reuse=True):
v = tf.get_variable("v", []) # "v" is alredy there, reused
losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
self.assertEqual(3, len(losses)) # No new loss added.
def loss(self, predicts, labels, objects_num):
"""Add Loss to all the trainable variables
Args:
predicts: 4-D tensor [batch_size, cell_size, cell_size, 5 * boxes_per_cell]
===> (num_classes, boxes_per_cell, 4 * boxes_per_cell)
labels : 3-D tensor of [batch_size, max_objects, 5]
objects_num: 1-D tensor [batch_size]
"""
class_loss = tf.constant(0, tf.float32)
object_loss = tf.constant(0, tf.float32)
noobject_loss = tf.constant(0, tf.float32)
coord_loss = tf.constant(0, tf.float32)
loss = [0, 0, 0, 0]
for i in range(self.batch_size):
predict = predicts[i, :, :, :]
label = labels[i, :, :]
object_num = objects_num[i]
nilboy = tf.ones([7,7,2])
tuple_results = tf.while_loop(self.cond1, self.body1, [tf.constant(0), object_num, [class_loss, object_loss, noobject_loss, coord_loss], predict, label, nilboy])
for j in range(4):
loss[j] = loss[j] + tuple_results[2][j]
nilboy = tuple_results[5]
tf.add_to_collection('losses', (loss[0] + loss[1] + loss[2] + loss[3])/self.batch_size)
tf.summary.scalar('class_loss', loss[0]/self.batch_size)
tf.summary.scalar('object_loss', loss[1]/self.batch_size)
tf.summary.scalar('noobject_loss', loss[2]/self.batch_size)
tf.summary.scalar('coord_loss', loss[3]/self.batch_size)
tf.summary.scalar('weight_loss', tf.add_n(tf.get_collection('losses')) - (loss[0] + loss[1] + loss[2] + loss[3])/self.batch_size )
return tf.add_n(tf.get_collection('losses'), name='total_loss'), nilboy
def optimize(nn_last_layer, correct_label, learning_rate, num_classes):
"""
Build the TensorFLow loss and optimizer operations.
:param nn_last_layer: TF Tensor of the last layer in the neural network
:param correct_label: TF Placeholder for the correct label image
:param learning_rate: TF Placeholder for the learning rate
:param num_classes: Number of classes to classify
:return: Tuple of (logits, train_op, cross_entropy_loss)
"""
# TODO: Implement function
logits = tf.reshape(nn_last_layer, (-1, num_classes))
labels = tf.reshape(correct_label, (-1, num_classes))
reg_loss = sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = labels))
total_loss = reg_loss + cross_entropy_loss
new_trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "train_scope")
trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)[-10-len(new_trainable_vars):]
print('2', trainable_vars)
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
# For passing unit test
if trainable_vars == []:
trainable_vars = None
train_op = optimizer.minimize(total_loss, var_list=trainable_vars)
return logits, train_op, total_loss
def testAddWeight(self):
with self.test_session():
layer = base_layers._Layer(name='my_layer')
# Test basic variable creation.
variable = layer._add_weight('my_var', [2, 2],
initializer=tf.zeros_initializer)
self.assertEqual(variable.name, 'my_var:0')
self.assertListEqual(layer.weights, [variable])
self.assertListEqual(layer.trainable_weights, [variable])
self.assertListEqual(layer.non_trainable_weights, [])
self.assertListEqual(layer.weights,
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
# Test non-trainable variable creation.
# layer._add_weight should work even outside `build` and `call`.
variable_2 = layer._add_weight('non_trainable_var', [2, 2],
initializer=tf.zeros_initializer,
trainable=False)
self.assertListEqual(layer.weights, [variable, variable_2])
self.assertListEqual(layer.trainable_weights, [variable])
self.assertListEqual(layer.non_trainable_weights, [variable_2])
self.assertEqual(
len(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)), 1)
# Test with regularizer.
regularizer = lambda x: tf.reduce_sum(x) * 1e-3
variable = layer._add_weight('reg_var', [2, 2],
initializer=tf.zeros_initializer,
regularizer=regularizer)
self.assertEqual(len(layer.losses), 1)
def testSaveAsText(self):
export_dir = os.path.join(
compat.as_bytes(tf.test.get_temp_dir()), compat.as_bytes("astext"))
builder = saved_model_builder.SavedModelBuilder(export_dir)
# Graph with a single variable. SavedModel invoked to:
# - add with weights.
with self.test_session(graph=tf.Graph()) as sess:
v = tf.Variable(42, name="v")
sess.run(tf.initialize_all_variables())
self.assertEqual(42, v.eval())
builder.add_meta_graph_and_variables(sess, ["foo"])
# Graph with the same single variable. SavedModel invoked to:
# - simply add the model (weights are not updated).
with self.test_session(graph=tf.Graph()) as sess:
v = tf.Variable(43, name="v")
sess.run(tf.initialize_all_variables())
self.assertEqual(43, v.eval())
builder.add_meta_graph(["bar"])
# Save the SavedModel to disk in text format.
builder.save(as_text=True)
# Restore the graph with tag "foo", whose variables were saved.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["foo"], export_dir)
self.assertEqual(42, tf.get_collection(tf.GraphKeys.VARIABLES)[0].eval())
# Restore the graph with tag "bar", whose variables were not saved.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["bar"], export_dir)
self.assertEqual(42, tf.get_collection(tf.GraphKeys.VARIABLES)[0].eval())
def __init__(self, params=params, dyn='FCC'):
tf.reset_default_graph()
data = self.sample_mog(params['batch_size'])
noise = ds.Normal(tf.zeros(params['z_dim']),
tf.ones(params['z_dim'])).sample(params['batch_size'])
# Construct generator and discriminator nets
with slim.arg_scope([slim.fully_connected], weights_initializer=tf.orthogonal_initializer(gain=1.4)):
samples = self.generator(noise, output_dim=params['x_dim'])
real_score = self.discriminator(data)
fake_score = self.discriminator(samples, reuse=True)
# Saddle objective
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=real_score, labels=tf.ones_like(real_score)) +
tf.nn.sigmoid_cross_entropy_with_logits(logits=fake_score, labels=tf.zeros_like(fake_score)))
gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator")
disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "discriminator")
gen_shapes = [tuple(v.get_shape().as_list()) for v in gen_vars]
disc_shapes = [tuple(v.get_shape().as_list()) for v in disc_vars]
# Generator gradient
g_opt = tf.train.GradientDescentOptimizer(learning_rate=params['gen_learning_rate'])
g_grads = g_opt.compute_gradients(-loss, var_list=gen_vars)
# Discriminator gradient
d_opt = tf.train.GradientDescentOptimizer(learning_rate=params['disc_learning_rate'])
d_grads = d_opt.compute_gradients(loss, var_list=disc_vars)
# Squared Norm of Gradient: d/dx 1/2||F||^2 = J^T F
grads_norm_sep = [tf.reduce_sum(g[0]**2) for g in g_grads+d_grads]
grads_norm = 0.5*tf.reduce_sum(grads_norm_sep)
# Gradient of Squared Norm
JTF = tf.gradients(grads_norm, xs=gen_vars+disc_vars)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
self.params = params
self.data = data
self.samples = samples
self.gen_vars = gen_vars
self.disc_vars = disc_vars
self.gen_shapes = gen_shapes
self.disc_shapes = disc_shapes
self.Fg = g_grads
self.Fd = d_grads
self.JTF = JTF
self.sess = sess
self.findiff_step = params['findiff_step']
self.gamma = params['gamma']
self.dyn = dyn
if dyn == 'FCC':
self.F = self.FCC
else:
self.F = self._F
def load(self, dir_name, epoch=0, name=None):
"""save model to dir
Parameters
----------
dir_name: str
name of the directory
epoch: int
"""
if name is None or name == self.name: # the name of saved model is the same as ours
dir_name = os.path.join(dir_name, self.name)
model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name)
saver = tf.train.Saver(model_vars)
saver.restore(self.sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch))
else: # load a checkpoint with different name
backup_graph = tf.get_default_graph()
kv_dict = {}
# load checkpoint from another saved graph
with tf.Graph().as_default(), tf.Session() as sess:
tf.train.import_meta_graph(os.path.join(dir_name, name, (self.subclass_name + "_%d") % epoch + ".meta"))
dir_name = os.path.join(dir_name, name)
model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(model_vars)
saver.restore(sess, os.path.join(dir_name, (self.subclass_name + "_%d") % epoch))
for item in tf.global_variables():
kv_dict[item.name] = sess.run(item)
# assign to now graph
backup_graph.as_default()
model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.name)
for item in model_vars:
old_name = item.name.replace(self.name, name)
self.sess.run(tf.assign(item, kv_dict[old_name]))
def combined_loss_G(self,batch_size_tf):
"""
Calculates the sum of the combined adversarial, lp and GDL losses in the given proportion. Used
for training the generative model.
@param gen_frames: A list of tensors of the generated frames at each scale.
@param gt_frames: A list of tensors of the ground truth frames at each scale.
@param d_preds: A list of tensors of the classifications made by the discriminator model at each
scale.
@param lam_adv: The percentage of the adversarial loss to use in the combined loss.
@param lam_lp: The percentage of the lp loss to use in the combined loss.
@param lam_gdl: The percentage of the GDL loss to use in the combined loss.
@param l_num: 1 or 2 for l1 and l2 loss, respectively).
@param alpha: The power to which each gradient term is raised in GDL loss.
@return: The combined adversarial, lp and GDL losses.
"""
diceterm=loss_dice(self.G, self.CT_GT, self.num_classes,batch_size_tf)
fcnterm=lossfcn(self.G, self.CT_GT, self.num_classes, batch_size_tf, self.classweights)
if self.adversarial:
bceterm=tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(self.D_logits_, tf.ones_like(self.D_)))
loss_=self.lam_dice*diceterm + self.lam_fcn*fcnterm + self.lam_adv*bceterm
tf.add_to_collection('losses', loss_)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
return loss, diceterm, fcnterm, bceterm
else:
loss_=self.lam_dice*diceterm + self.lam_fcn*fcnterm
tf.add_to_collection('losses', loss_)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
return loss, self.lam_dice*diceterm, self.lam_fcn*fcnterm
def __init__(self, a_dim, s_dim, a_bound,):
self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim + 1), dtype=np.float32)
self.pointer = 0
self.sess = tf.Session()
self.a_replace_counter, self.c_replace_counter = 0, 0
self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound,
self.S = tf.placeholder(tf.float32, [None, s_dim], 's')
self.S_ = tf.placeholder(tf.float32, [None, s_dim], 's_')
self.R = tf.placeholder(tf.float32, [None, 1], 'r')
with tf.variable_scope('Actor'):
self.a = self._build_a(self.S, scope='eval', trainable=True)
a_ = self._build_a(self.S_, scope='target', trainable=False)
with tf.variable_scope('Critic'):
# assign self.a = a in memory when calculating q for td_error,
# otherwise the self.a is from Actor when updating Actor
q = self._build_c(self.S, self.a, scope='eval', trainable=True)
q_ = self._build_c(self.S_, a_, scope='target', trainable=False)
# networks parameters
self.ae_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/eval')
self.at_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/target')
self.ce_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval')
self.ct_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target')
q_target = self.R + GAMMA * q_
# in the feed_dic for the td_error, the self.a should change to actions in memory
td_error = tf.losses.mean_squared_error(labels=q_target, predictions=q)
self.ctrain = tf.train.AdamOptimizer(LR_C).minimize(td_error, var_list=self.ce_params)
a_loss = - tf.reduce_mean(q) # maximize the q
self.atrain = tf.train.AdamOptimizer(LR_A).minimize(a_loss, var_list=self.ae_params)
self.sess.run(tf.global_variables_initializer())
def loss(y, model_vars, Y, l2_reg, scope=None):
"""
L2-loss model on top of the network raw output.
Args:
y: network output tensor
model_vars: [w_conv, thresh, w_e, w_s, w_d]
Y: ground truth tensor
l2_reg: l2 regularization strength
scope: unique prefix string identifying the tower, e.g. 'tower_00'
Returns:
total_loss: total loss Tensor
"""
sq_loss = tf.nn.l2_loss(y - Y, name='sq_loss')
tf.add_to_collection('losses', sq_loss)
if l2_reg > 0:
with tf.name_scope('l2_decay'):
w_conv, thresh, w_e, w_s, w_d = model_vars
for decay_var in [w_conv, w_e, w_s, w_d]:
weight_decay = tf.mul(tf.nn.l2_loss(decay_var), l2_reg)
tf.add_to_collection('losses', weight_decay)
total_loss = tf.add_n(tf.get_collection('losses', scope=scope), name='total_loss')
# Add loss summaries
for loss in tf.get_collection('losses', scope=scope) + [total_loss]:
loss_name = re.sub('%s_[0-9]*/' % FLAGS.tower_name, '', loss.op.name)
tf.scalar_summary(loss_name, loss)
return total_loss
def __init__(self, sess, state_dim, action_dim, learning_rate, gamma, t_replace_iter, a, a_):
self.sess = sess
self.s_dim = state_dim
self.a_dim = action_dim
self.lr = learning_rate
self.gamma = gamma
self.t_replace_iter = t_replace_iter
self.t_replace_counter = 0
with tf.variable_scope('Critic'):
# Input (s, a), output q
self.a = a
self.q = self._build_net(S, self.a, 'eval_net', trainable=True)
# Input (s_, a_), output q_ for q_target
self.q_ = self._build_net(S_, a_, 'target_net', trainable=False) # target_q is based on a_ from Actor's target_net
self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/eval_net')
self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/target_net')
with tf.variable_scope('target_q'):
self.target_q = R + self.gamma * self.q_
with tf.variable_scope('TD_error'):
self.loss = tf.reduce_mean(tf.squared_difference(self.target_q, self.q))
with tf.variable_scope('C_train'):
self.train_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss)
with tf.variable_scope('a_grad'):
self.a_grads = tf.gradients(self.q, a)[0] # tensor of gradients of each sample (None, a_dim)
def testCheckStatsDouble(self, dtype):
"""The correct statistics are being computed for double connection.
Connected in parallel, it's ill-defined what order the updates will happen
in. A double update could happen, or two sequential updates. E.g. If
decay_rate is 0.9, the start value is 1.0, and the target value is 0.0, the
value could progress as
1.00 -> 0.90 -> 0.81,
if the second update uses the fresh second value. Or as
1.00 -> 0.90 -> 0.80
if the second update uses the stale first value.
We fix this here by running them in sequential run calls to ensure that this
test is deterministic.
The two situations are minimally different, especially if decay_rate is
close to one (e.g. the default of 0.999).
Args:
dtype: TensorFlow datatype of input test batch.
"""
v, _, inputs = self._get_inputs(dtype)
bn = snt.BatchNorm(offset=False, scale=False, decay_rate=0.9)
with tf.name_scope("net1"):
bn(inputs, is_training=True)
with tf.name_scope("net2"):
bn(inputs, is_training=True)
update_ops_1 = tuple(tf.get_collection(tf.GraphKeys.UPDATE_OPS, "net1"))
self.assertEqual(len(update_ops_1), 2)
update_ops_2 = tuple(tf.get_collection(tf.GraphKeys.UPDATE_OPS, "net2"))
self.assertEqual(len(update_ops_2), 2)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
mm, mv = sess.run([bn.moving_mean, bn.moving_variance])
self.assertAllClose(np.zeros([1, 6]), mm)
self.assertAllClose(np.ones([1, 6]), mv)
sess.run(update_ops_1)
sess.run(update_ops_2)
mm, mv = sess.run([bn.moving_mean,
bn.moving_variance])
correct_mm = (1.0 - bn._decay_rate) * v
correct_mm = (1.0 - bn._decay_rate) * v + bn._decay_rate * correct_mm
correct_mv = np.ones([1, 6]) * bn._decay_rate**2
self.assertAllClose(np.reshape(correct_mm, [1, 6]), mm)
self.assertAllClose(np.reshape(correct_mv, [1, 6]), mv)
def __init__(self, a_dim, s_dim, a_bound,):
self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim + 1), dtype=np.float32)
self.pointer = 0
self.sess = tf.Session()
self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound,
self.S = tf.placeholder(tf.float32, [None, s_dim], 's')
self.S_ = tf.placeholder(tf.float32, [None, s_dim], 's_')
self.R = tf.placeholder(tf.float32, [None, 1], 'r')
self.a = self._build_a(self.S,)
q = self._build_c(self.S, self.a, )
a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Actor')
c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Critic')
ema = tf.train.ExponentialMovingAverage(decay=1 - TAU) # soft replacement
def ema_getter(getter, name, *args, **kwargs):
return ema.average(getter(name, *args, **kwargs))
target_update = [ema.apply(a_params), ema.apply(c_params)] # soft update operation
a_ = self._build_a(self.S_, reuse=True, custom_getter=ema_getter) # replaced target parameters
q_ = self._build_c(self.S_, a_, reuse=True, custom_getter=ema_getter)
a_loss = - tf.reduce_mean(q) # maximize the q
self.atrain = tf.train.AdamOptimizer(LR_A).minimize(a_loss, var_list=a_params)
with tf.control_dependencies(target_update): # soft replacement happened at here
q_target = self.R + GAMMA * q_
td_error = tf.losses.mean_squared_error(labels=q_target, predictions=q)
self.ctrain = tf.train.AdamOptimizer(LR_C).minimize(td_error, var_list=c_params)
self.sess.run(tf.global_variables_initializer())
def _build(self):
saver = tf.train.import_meta_graph(self._ckpt_path + '.meta')
saver.restore(
sess=self._session,
save_path=self._ckpt_path)
self._action = tf.get_collection('action_op')[0]
self._obs = tf.get_collection('observation_placeholder')[0]
def freeze(self):
"""Useful for e.g. deploying model on website.
Args: directory containing model ckpt files we'd like to freeze.
"""
if not tf.get_collection('freezer'):
self.log.warning('No freezer found. Not saving a frozen model.')
return
# Note: output_node_names is only used to tell tensorflow what is can
# throw away in the frozen graph (e.g. training ops).
output_node_names = ",".join(
[t.name.rstrip(':0') for t in tf.get_collection('freezer')])
self.log.info('Output node names: %r', output_node_names)
# Save a graph with only the bare necessities for chat sessions.
output_graph_def = tf.graph_util.convert_variables_to_constants(
self.sess, self.graph.as_graph_def(), output_node_names.split(','))
output_fname = os.path.join(self.ckpt_dir, "frozen_model.pb")
with tf.gfile.GFile(output_fname, 'wb') as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
subprocess.call(['cp', self.dataset.paths['vocab'], self.ckpt_dir])
def __model_gradients(variable_scope: tf.VariableScope,
transformation_variable_scope: tf.VariableScope,
output: tf.Tensor, output_gradient: tf.Tensor):
trainable_variables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
transformation_variable_scope.name)
gradients = tf.gradients(output, trainable_variables, output_gradient)
for gradient in gradients:
gradient_accumulator = tf.Variable(tf.zeros(
gradient.get_shape(), gradient.dtype), name="gradient_accumulator")
tf.add_to_collection(
'{}/model_gradients'.format(variable_scope.name),
gradient)
tf.add_to_collection(
'{}/model_gradient_accumulators'.format(variable_scope.name),
gradient_accumulator)
tf.add_to_collection(
'{}/update_model_gradient_accumulators'.format(
variable_scope.name),
tf.assign_add(gradient_accumulator, gradient).op)
with tf.control_dependencies(tf.get_collection(
"{}/update_model_gradient_accumulators".format(variable_scope.name))):
# there is no noop
tf.add(1, 1, "update_model_gradient_accumulators")
tf.variables_initializer(
tf.get_collection(
'{}/model_gradient_accumulators'.format(variable_scope.name)),
'zero_model_gradient_accumulators')
def add_optimization(learning_rate, beta1, beta2, disc_gen, disc_true,
gen_label, disc_label):
gen_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
disc_gen, tf.ones_like(disc_gen)), name='gen_loss')
disc_g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
disc_gen, tf.zeros_like(disc_gen)))
disc_x_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
disc_true, tf.ones_like(disc_true)))
disc_loss = tf.add(disc_g_loss, disc_x_loss, name='disc_loss')
gen_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=gen_label)
disc_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=disc_label)
# print 'gen vars---------------------'
# for v in gen_vars:
# print v.name
# print 'disc vars----------------'
# for v in disc_vars:
# print v.name
gen_opt = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=beta1,
beta2=beta2).minimize(gen_loss,
var_list=gen_vars)
disc_opt = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=beta1,
beta2=beta2).minimize(disc_loss,
var_list=disc_vars)
return gen_loss, disc_loss, gen_opt, disc_opt
def testCustomMainOp(self):
export_dir = os.path.join(tf.test.get_temp_dir(), "test_main_op")
builder = saved_model_builder.SavedModelBuilder(export_dir)
with self.test_session(graph=tf.Graph()) as sess:
# Add `v1` and `v2` variables to the graph.
v1 = tf.Variable(1, name="v1")
tf.add_to_collection("v", v1)
v2 = tf.Variable(2, name="v2")
tf.add_to_collection("v", v2)
# Initialize another variable `v3` to 42.
v3 = tf.Variable(42, name="v3")
tf.add_to_collection("v", v3)
# Set up an assignment op to be run as part of the main_op.
with tf.control_dependencies([main_op.main_op()]):
add_v1_v2 = tf.add(v1._ref(), v2._ref())
custom_main_op = tf.group(tf.assign(v3, add_v1_v2))
sess.run(custom_main_op)
builder.add_meta_graph_and_variables(
sess, ["foo"], main_op=custom_main_op)
# Save the SavedModel to disk.
builder.save()
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["foo"], export_dir)
self.assertEqual(1, tf.get_collection("v")[0].eval())
self.assertEqual(2, tf.get_collection("v")[1].eval())
# Evaluates to the sum of the first two variables and assigned as part of
# the main_op, following a restore.
self.assertEqual(3, tf.get_collection("v")[2].eval())
def testTags(self):
export_dir = os.path.join(tf.test.get_temp_dir(), "test_tags")
builder = saved_model_builder.SavedModelBuilder(export_dir)
# Graph with a single variable. SavedModel invoked to:
# - add with weights.
# - a single tag (from predefined constants).
with self.test_session(graph=tf.Graph()) as sess:
self._init_and_validate_variable(sess, "v", 42)
builder.add_meta_graph_and_variables(sess, [tag_constants.TRAINING])
# Graph that updates the single variable. SavedModel invoked to:
# - simply add the model (weights are not updated).
# - a single tag (from predefined constants).
with self.test_session(graph=tf.Graph()) as sess:
self._init_and_validate_variable(sess, "v", 43)
builder.add_meta_graph([tag_constants.SERVING])
# Graph that updates the single variable. SavedModel is invoked:
# - to add the model (weights are not updated).
# - multiple custom tags.
with self.test_session(graph=tf.Graph()) as sess:
self._init_and_validate_variable(sess, "v", 44)
builder.add_meta_graph(["foo", "bar"])
# Save the SavedModel to disk.
builder.save()
# Restore the graph with a single predefined tag whose variables were saved.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, [tag_constants.TRAINING], export_dir)
self.assertEqual(
42, tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0].eval())
# Restore the graph with a single predefined tag whose variables were not
# saved.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, [tag_constants.SERVING], export_dir)
self.assertEqual(
42, tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0].eval())
# Restore the graph with multiple tags. Provide duplicate tags to test set
# semantics.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["foo", "bar", "foo"], export_dir)
self.assertEqual(
42, tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0].eval())
# Try restoring a graph with a non-existent tag. This should yield a runtime
# error.
with self.test_session(graph=tf.Graph()) as sess:
self.assertRaises(RuntimeError, loader.load, sess, ["INVALID"],
export_dir)
# Try restoring a graph where a subset of the tags match. Since tag matching
# for meta graph defs follows "all" semantics, this should yield a runtime
# error.
with self.test_session(graph=tf.Graph()) as sess:
self.assertRaises(RuntimeError, loader.load, sess, ["foo", "baz"],
export_dir)
def testSaveAsText(self):
export_dir = os.path.join(tf.test.get_temp_dir(), "test_astext")
builder = saved_model_builder.SavedModelBuilder(export_dir)
# Graph with a single variable. SavedModel invoked to:
# - add with weights.
with self.test_session(graph=tf.Graph()) as sess:
self._init_and_validate_variable(sess, "v", 42)
builder.add_meta_graph_and_variables(sess, ["foo"])
# Graph with the same single variable. SavedModel invoked to:
# - simply add the model (weights are not updated).
with self.test_session(graph=tf.Graph()) as sess:
self._init_and_validate_variable(sess, "v", 43)
builder.add_meta_graph(["bar"])
# Save the SavedModel to disk in text format.
builder.save(as_text=True)
# Restore the graph with tag "foo", whose variables were saved.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["foo"], export_dir)
self.assertEqual(
42, tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0].eval())
# Restore the graph with tag "bar", whose variables were not saved.
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["bar"], export_dir)
self.assertEqual(
42, tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0].eval())
def testmodel1():
print "\033[1;31mbegin load model>>>\033[0m"
with tf.Session() as sess:
new_saver = tf.train.import_meta_graph("saver_checkpoint.meta") # 恢复图模型
new_saver.restore(sess, "saver_checkpoint") # 恢复数据
# tf.get_collection() returns a list. In this example we only want the first one.
predict = tf.get_collection("predict")[0]
x = tf.get_collection("x")[0]
y_ = tf.get_collection("y_")[0]
keep_prob = tf.get_collection("keep_prob")[0]
# 单值预测
print "\033[1;31msingle predict\033[0m"
test = test_samples[1,].reshape(1, 784)
mark = np.diag([1] * 4)
prev = sess.run(predict, feed_dict={x: test, y_: mark, keep_prob: 1.0})
print u"[prev]:", chr(prev.tolist().index(1) + 65)
# 混淆矩阵测试
print "\033[1;31mbatch matrix\033[0m"
pre_labels = []
for sample in test_samples.tolist():
sample = np.array(sample)
pre_label = sess.run(predict, feed_dict={x: sample.reshape(1, 784), y_: mark, keep_prob: 1.0})
pre_labels.append(pre_label)
pre_char_labels = [chr(l.tolist().index(True) + 65) for l in pre_labels] # 预测值按真假分
test_char_labels = [chr(l.tolist().index(1) + 65) for l in test_labels] # 实际这是01分
print metrics.confusion_matrix(pre_char_labels, test_char_labels)
print "\033[1;31mpredict done!\033[0m"
def _update_network(self, trainer):
self.actions = tf.placeholder(shape=[None], dtype=tf.int32)
self.actions_onehot = tf.one_hot(
self.actions, self.a_dim, dtype=tf.float32)
self.target_v = tf.placeholder(shape=[None], dtype=tf.float32)
self.advantages = tf.placeholder(shape=[None], dtype=tf.float32)
self.outputs = tf.reduce_sum(
self.policy * self.actions_onehot, [1])
# loss
self.value_loss = 0.5 * tf.reduce_sum(tf.square(
self.target_v - tf.reshape(self.value, [-1])))
# higher entropy -> lower loss -> encourage exploration
self.entropy = -tf.reduce_sum(self.policy * tf.log(self.policy))
self.policy_loss = -tf.reduce_sum(
tf.log(self.outputs) * self.advantages)
self.loss = 0.5 * self.value_loss \
+ self.policy_loss - 0.01 * self.entropy
# local gradients
local_vars = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, self.scope)
self.gradients = tf.gradients(self.loss, local_vars)
self.var_norms = tf.global_norm(local_vars)
# grads[i] * clip_norm / max(global_norm, clip_norm)
grads, self.grad_norms = tf.clip_by_global_norm(self.gradients, 40.0)
# apply gradients to global network
global_vars = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, 'global')
self.apply_grads = trainer.apply_gradients(zip(grads, global_vars))
def __init_output(self):
with tf.variable_scope('output'):
# Losses
self.regularization_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
self.cross_entropy_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.y, name='loss'))
self.loss = self.regularization_loss + self.cross_entropy_loss
# Optimizer
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.args.learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
# This is for debugging NaNs. Check TensorFlow documentation.
self.check_op = tf.add_check_numerics_ops()
# Output and Metrics
self.y_out_softmax = tf.nn.softmax(self.logits)# softmax 归一化分类
self.y_out_argmax = tf.argmax(self.y_out_softmax, axis=-1, output_type=tf.int32)# 最大值得到分类结果
self.accuracy = tf.reduce_mean(tf.cast(tf.equal(self.y, self.y_out_argmax), tf.float32))#准确度
# 记录参数
with tf.name_scope('train-summary-per-iteration'):
tf.summary.scalar('loss', self.loss)
tf.summary.scalar('acc', self.accuracy)
self.summaries_merged = tf.summary.merge_all()
def __init__(self,
n_actions,
n_features,
learning_rate=0.01,
reward_decay=0.9, # gamma
epsilon_greedy=0.9, # epsilon
epsilon_increment = 0.001,
replace_target_iter=300, # 更新target网络的间隔步数
buffer_size=500, # 样本缓冲区
batch_size=32,
):
self.n_actions = n_actions
self.n_features = n_features
self.lr = learning_rate
self.gamma = reward_decay
self.epsilon_max = epsilon_greedy
self.replace_target_iter = replace_target_iter
self.buffer_size = buffer_size
self.buffer_counter = 0 # 统计目前进入过buffer的数量
self.batch_size = batch_size
self.epsilon = 0 if epsilon_increment is not None else epsilon_greedy
self.epsilon_max = epsilon_greedy
self.epsilon_increment = epsilon_increment
self.learn_step_counter = 0 # 学习计步器
self.buffer = np.zeros((self.buffer_size, n_features * 2 + 2)) # 初始化Experience buffer[s,a,r,s_]
self.build_net()
# 将eval网络中参数全部更新到target网络
target_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='target_net')
eval_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='eval_net')
with tf.variable_scope('soft_replacement'):
self.target_replace_op = [tf.assign(t, e) for t, e in zip(target_params, eval_params)]
self.sess = tf.Session()
tf.summary.FileWriter('logs/', self.sess.graph)
self.sess.run(tf.global_variables_initializer())
def testLegacyInitOp(self):
export_dir = os.path.join(tf.test.get_temp_dir(), "test_legacy_init_op")
builder = saved_model_builder.SavedModelBuilder(export_dir)
with self.test_session(graph=tf.Graph()) as sess:
# Add `v1` and `v2` variables to the graph.
v1 = tf.Variable(1, name="v1")
tf.add_to_collection("v", v1)
v2 = tf.Variable(2, name="v2")
tf.add_to_collection("v", v2)
# Initialize another variable `v3` to 42.
v3 = tf.Variable(42, name="v3", trainable=False, collections=[])
tf.add_to_collection("v", v3)
# Set up an assignment op to be run as part of the legacy_init_op.
assign_v3 = tf.assign(v3, tf.add(v1, v2))
legacy_init_op = tf.group(assign_v3, name="legacy_init_op")
sess.run(tf.initialize_all_variables())
builder.add_meta_graph_and_variables(sess, ["foo"], legacy_init_op=legacy_init_op)
# Save the SavedModel to disk.
builder.save()
with self.test_session(graph=tf.Graph()) as sess:
loader.load(sess, ["foo"], export_dir)
self.assertEqual(1, tf.get_collection("v")[0].eval())
self.assertEqual(2, tf.get_collection("v")[1].eval())
# Evaluates to the sum of the first two variables and assigned as part of
# the legacy_init_op, following a restore.
self.assertEqual(3, tf.get_collection("v")[2].eval())
def main_fun(argv, ctx):
import tensorflow as tf
from tensorflow.python.ops import control_flow_ops
from datasets import dataset_factory
from deployment import model_deploy
from nets import nets_factory
from preprocessing import preprocessing_factory
sys.argv = argv
slim = tf.contrib.slim
tf.app.flags.DEFINE_integer('num_gpus', '1',
'The number of GPUs to use per node')
tf.app.flags.DEFINE_boolean('rdma', False, 'Whether to use rdma.')
tf.app.flags.DEFINE_string('master', '',
'The address of the TensorFlow master to use.')
tf.app.flags.DEFINE_string(
'train_dir', '/tmp/tfmodel/',
'Directory where checkpoints and event logs are written to.')
tf.app.flags.DEFINE_integer('num_clones', 1,
'Number of model clones to deploy.')
tf.app.flags.DEFINE_boolean('clone_on_cpu', False,
'Use CPUs to deploy clones.')
tf.app.flags.DEFINE_integer('worker_replicas', 1,
'Number of worker replicas.')
tf.app.flags.DEFINE_integer(
'num_ps_tasks', 0,
'The number of parameter servers. If the value is 0, then the parameters '
'are handled locally by the worker.')
tf.app.flags.DEFINE_integer(
'num_readers', 4,
'The number of parallel readers that read data from the dataset.')
tf.app.flags.DEFINE_integer(
'num_preprocessing_threads', 4,
'The number of threads used to create the batches.')
tf.app.flags.DEFINE_integer('log_every_n_steps', 10,
'The frequency with which logs are print.')
tf.app.flags.DEFINE_integer(
'save_summaries_secs', 600,
'The frequency with which summaries are saved, in seconds.')
tf.app.flags.DEFINE_integer(
'save_interval_secs', 600,
'The frequency with which the model is saved, in seconds.')
tf.app.flags.DEFINE_integer(
'task', 0, 'Task id of the replica running the training.')
######################
# Optimization Flags #
######################
tf.app.flags.DEFINE_float('weight_decay', 0.00004,
'The weight decay on the model weights.')
tf.app.flags.DEFINE_string(
'optimizer', 'rmsprop',
'The name of the optimizer, one of "adadelta", "adagrad", "adam",'
'"ftrl", "momentum", "sgd" or "rmsprop".')
tf.app.flags.DEFINE_float('adadelta_rho', 0.95,
'The decay rate for adadelta.')
tf.app.flags.DEFINE_float('adagrad_initial_accumulator_value', 0.1,
'Starting value for the AdaGrad accumulators.')
tf.app.flags.DEFINE_float(
'adam_beta1', 0.9,
'The exponential decay rate for the 1st moment estimates.')
tf.app.flags.DEFINE_float(
'adam_beta2', 0.999,
'The exponential decay rate for the 2nd moment estimates.')
tf.app.flags.DEFINE_float('opt_epsilon', 1.0,
'Epsilon term for the optimizer.')
tf.app.flags.DEFINE_float('ftrl_learning_rate_power', -0.5,
'The learning rate power.')
tf.app.flags.DEFINE_float('ftrl_initial_accumulator_value', 0.1,
'Starting value for the FTRL accumulators.')
tf.app.flags.DEFINE_float('ftrl_l1', 0.0,
'The FTRL l1 regularization strength.')
tf.app.flags.DEFINE_float('ftrl_l2', 0.0,
'The FTRL l2 regularization strength.')
tf.app.flags.DEFINE_float(
'momentum', 0.9,
'The momentum for the MomentumOptimizer and RMSPropOptimizer.')
tf.app.flags.DEFINE_float('rmsprop_momentum', 0.9, 'Momentum.')
tf.app.flags.DEFINE_float('rmsprop_decay', 0.9, 'Decay term for RMSProp.')
#######################
# Learning Rate Flags #
#######################
tf.app.flags.DEFINE_string(
'learning_rate_decay_type', 'exponential',
'Specifies how the learning rate is decayed. One of "fixed", "exponential",'
' or "polynomial"')
tf.app.flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
tf.app.flags.DEFINE_float(
'end_learning_rate', 0.0001,
'The minimal end learning rate used by a polynomial decay learning rate.'
)
tf.app.flags.DEFINE_float('label_smoothing', 0.0,
'The amount of label smoothing.')
tf.app.flags.DEFINE_float('learning_rate_decay_factor', 0.94,
'Learning rate decay factor.')
tf.app.flags.DEFINE_float(
'num_epochs_per_decay', 2.0,
'Number of epochs after which learning rate decays.')
tf.app.flags.DEFINE_bool(
'sync_replicas', False,
'Whether or not to synchronize the replicas during training.')
tf.app.flags.DEFINE_integer(
'replicas_to_aggregate', 1,
'The Number of gradients to collect before updating params.')
tf.app.flags.DEFINE_float(
'moving_average_decay', None,
'The decay to use for the moving average.'
'If left as None, then moving averages are not used.')
#######################
# Dataset Flags #
#######################
tf.app.flags.DEFINE_string('dataset_name', 'imagenet',
'The name of the dataset to load.')
tf.app.flags.DEFINE_string('dataset_split_name', 'train',
'The name of the train/test split.')
tf.app.flags.DEFINE_string(
'dataset_dir', None,
'The directory where the dataset files are stored.')
tf.app.flags.DEFINE_integer(
'labels_offset', 0,
'An offset for the labels in the dataset. This flag is primarily used to '
'evaluate the VGG and ResNet architectures which do not use a background '
'class for the ImageNet dataset.')
tf.app.flags.DEFINE_string('model_name', 'inception_v3',
'The name of the architecture to train.')
tf.app.flags.DEFINE_string(
'preprocessing_name', None,
'The name of the preprocessing to use. If left '
'as `None`, then the model_name flag is used.')
tf.app.flags.DEFINE_integer('batch_size', 32,
'The number of samples in each batch.')
tf.app.flags.DEFINE_integer('train_image_size', None, 'Train image size')
tf.app.flags.DEFINE_integer('max_number_of_steps', None,
'The maximum number of training steps.')
#####################
# Fine-Tuning Flags #
#####################
tf.app.flags.DEFINE_string(
'checkpoint_path', None,
'The path to a checkpoint from which to fine-tune.')
tf.app.flags.DEFINE_string(
'checkpoint_exclude_scopes', None,
'Comma-separated list of scopes of variables to exclude when restoring '
'from a checkpoint.')
tf.app.flags.DEFINE_string(
'trainable_scopes', None,
'Comma-separated list of scopes to filter the set of variables to train.'
'By default, None would train all the variables.')
tf.app.flags.DEFINE_boolean(
'ignore_missing_vars', False,
'When restoring a checkpoint would ignore missing variables.')
FLAGS = tf.app.flags.FLAGS
FLAGS.job_name = ctx.job_name
FLAGS.task = ctx.task_index
FLAGS.num_clones = FLAGS.num_gpus
FLAGS.worker_replicas = len(ctx.cluster_spec['worker'])
assert (FLAGS.num_ps_tasks == (len(ctx.cluster_spec['ps'])
if 'ps' in ctx.cluster_spec else 0))
def _configure_learning_rate(num_samples_per_epoch, global_step):
"""Configures the learning rate.
Args:
num_samples_per_epoch: The number of samples in each epoch of training.
global_step: The global_step tensor.
Returns:
A `Tensor` representing the learning rate.
Raises:
ValueError: if
"""
decay_steps = int(num_samples_per_epoch / FLAGS.batch_size *
FLAGS.num_epochs_per_decay)
if FLAGS.sync_replicas:
decay_steps /= FLAGS.replicas_to_aggregate
if FLAGS.learning_rate_decay_type == 'exponential':
return tf.train.exponential_decay(
FLAGS.learning_rate,
global_step,
decay_steps,
FLAGS.learning_rate_decay_factor,
staircase=True,
name='exponential_decay_learning_rate')
elif FLAGS.learning_rate_decay_type == 'fixed':
return tf.constant(FLAGS.learning_rate, name='fixed_learning_rate')
elif FLAGS.learning_rate_decay_type == 'polynomial':
return tf.train.polynomial_decay(
FLAGS.learning_rate,
global_step,
decay_steps,
FLAGS.end_learning_rate,
power=1.0,
cycle=False,
name='polynomial_decay_learning_rate')
else:
raise ValueError(
'learning_rate_decay_type [%s] was not recognized',
FLAGS.learning_rate_decay_type)
def _configure_optimizer(learning_rate):
"""Configures the optimizer used for training.
Args:
learning_rate: A scalar or `Tensor` learning rate.
Returns:
An instance of an optimizer.
Raises:
ValueError: if FLAGS.optimizer is not recognized.
"""
if FLAGS.optimizer == 'adadelta':
optimizer = tf.train.AdadeltaOptimizer(learning_rate,
rho=FLAGS.adadelta_rho,
epsilon=FLAGS.opt_epsilon)
elif FLAGS.optimizer == 'adagrad':
optimizer = tf.train.AdagradOptimizer(
learning_rate,
initial_accumulator_value=FLAGS.
adagrad_initial_accumulator_value)
elif FLAGS.optimizer == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate,
beta1=FLAGS.adam_beta1,
beta2=FLAGS.adam_beta2,
epsilon=FLAGS.opt_epsilon)
elif FLAGS.optimizer == 'ftrl':
optimizer = tf.train.FtrlOptimizer(
learning_rate,
learning_rate_power=FLAGS.ftrl_learning_rate_power,
initial_accumulator_value=FLAGS.ftrl_initial_accumulator_value,
l1_regularization_strength=FLAGS.ftrl_l1,
l2_regularization_strength=FLAGS.ftrl_l2)
elif FLAGS.optimizer == 'momentum':
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum=FLAGS.momentum,
name='Momentum')
elif FLAGS.optimizer == 'rmsprop':
optimizer = tf.train.RMSPropOptimizer(
learning_rate,
decay=FLAGS.rmsprop_decay,
momentum=FLAGS.rmsprop_momentum,
epsilon=FLAGS.opt_epsilon)
elif FLAGS.optimizer == 'sgd':
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
else:
raise ValueError('Optimizer [%s] was not recognized',
FLAGS.optimizer)
return optimizer
def _add_variables_summaries(learning_rate):
summaries = []
for variable in slim.get_model_variables():
summaries.append(tf.summary.histogram(variable.op.name, variable))
summaries.append(
tf.summary.scalar('training/Learning Rate', learning_rate))
return summaries
def _get_init_fn():
"""Returns a function run by the chief worker to warm-start the training.
Note that the init_fn is only run when initializing the model during the very
first global step.
Returns:
An init function run by the supervisor.
"""
if FLAGS.checkpoint_path is None:
return None
# Warn the user if a checkpoint exists in the train_dir. Then we'll be
# ignoring the checkpoint anyway.
if tf.train.latest_checkpoint(FLAGS.train_dir):
tf.logging.info(
'Ignoring --checkpoint_path because a checkpoint already exists in %s'
% FLAGS.train_dir)
return None
exclusions = []
if FLAGS.checkpoint_exclude_scopes:
exclusions = [
scope.strip()
for scope in FLAGS.checkpoint_exclude_scopes.split(',')
]
# TODO(sguada) variables.filter_variables()
variables_to_restore = []
for var in slim.get_model_variables():
excluded = False
for exclusion in exclusions:
if var.op.name.startswith(exclusion):
excluded = True
break
if not excluded:
variables_to_restore.append(var)
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Fine-tuning from %s' % checkpoint_path)
return slim.assign_from_checkpoint_fn(
checkpoint_path,
variables_to_restore,
ignore_missing_vars=FLAGS.ignore_missing_vars)
def _get_variables_to_train():
"""Returns a list of variables to train.
Returns:
A list of variables to train by the optimizer.
"""
if FLAGS.trainable_scopes is None:
return tf.trainable_variables()
else:
scopes = [
scope.strip() for scope in FLAGS.trainable_scopes.split(',')
]
variables_to_train = []
for scope in scopes:
variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope)
variables_to_train.extend(variables)
return variables_to_train
# main
cluster_spec, server = TFNode.start_cluster_server(ctx, FLAGS.num_gpus)
if ctx.job_name == 'ps':
# `ps` jobs wait for incoming connections from the workers.
server.join()
else:
# `worker` jobs will actually do the work.
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
######################
# Config model_deploy#
######################
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.worker_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
####################
# Select the network #
####################
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=True)
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
with tf.device(deploy_config.inputs_device()):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size)
[image, label] = provider.get(['image', 'label'])
label -= FLAGS.labels_offset
train_image_size = FLAGS.train_image_size or network_fn.default_image_size
image = image_preprocessing_fn(image, train_image_size,
train_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
labels = slim.one_hot_encoding(
labels, dataset.num_classes - FLAGS.labels_offset)
batch_queue = slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * deploy_config.num_clones)
####################
# Define the model #
####################
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple clones of network_fn."""
images, labels = batch_queue.dequeue()
logits, end_points = network_fn(images)
#############################
# Specify the loss function #
#############################
if 'AuxLogits' in end_points:
slim.losses.softmax_cross_entropy(
end_points['AuxLogits'],
labels,
label_smoothing=FLAGS.label_smoothing,
weights=0.4,
scope='aux_loss')
slim.losses.softmax_cross_entropy(
logits,
labels,
label_smoothing=FLAGS.label_smoothing,
weights=1.0)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(
tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES,
first_clone_scope):
summaries.add(
tf.summary.scalar('losses/%s' % loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
#########################################
# Configure the optimization procedure. #
#########################################
with tf.device(deploy_config.optimizer_device()):
learning_rate = _configure_learning_rate(
dataset.num_samples, global_step)
optimizer = _configure_optimizer(learning_rate)
summaries.add(tf.summary.scalar('learning_rate',
learning_rate))
if FLAGS.sync_replicas:
# If sync_replicas is enabled, the averaging will be done in the chief
# queue runner.
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
variable_averages=variable_averages,
variables_to_average=moving_average_variables,
replica_id=tf.constant(FLAGS.task, tf.int32, shape=()),
total_num_replicas=FLAGS.worker_replicas)
elif FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = _get_variables_to_train()
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
###########################
# Kicks off the training. #
###########################
summary_writer = tf.summary.FileWriter(
"tensorboard_%d" % (ctx.worker_num),
graph=tf.get_default_graph())
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master=server.target,
is_chief=(FLAGS.task == 0),
init_fn=_get_init_fn(),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
save_interval_secs=FLAGS.save_interval_secs,
summary_writer=summary_writer,
sync_optimizer=optimizer if FLAGS.sync_replicas else None)
tf.square(V_batch), [1, 2]) - tf.reduce_sum(
tf.square(VY_batch), [1, 2]) - tf.reduce_sum(
tf.square(YV_batch), [1, 2]) + tf.reduce_sum(
tf.square(Y_batch), [1, 2])
mean_cost = 10000 * tf.reduce_mean(cost_per_sample)
return mean_cost
# Definition of the network and loss
assign, cost_pit, w_seq = assignment(y1r, y1i, y2r, y2i, est1, est2, seq_len)
embed = tcn(est1, est2, xr, xi, seq_len, keep_prob)
cost = seq_cost(embed, assign, w_seq, seq_len)
# Define loss and optimizer
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
if FLAGS.is_adam == 1:
optimizer = tf.train.AdamOptimizer(
learning_rate=lr, beta2=0.9).minimize(cost) # Adam Optimizer
else:
optimizer = tf.train.MomentumOptimizer(learning_rate=lr,
momentum=0.9).minimize(
cost) # Momentum Optimizer
# Model dir and model saver
model_dir = os.getcwd(
) + "/exp/" + FLAGS.exp_name + "/models/" + FLAGS.time_stamp
if not os.path.exists(model_dir):
os.makedirs(model_dir)
saver = tf.train.Saver(max_to_keep=None)
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# placeholders
lr = tf.placeholder(tf.float32, name='learning_rate')
images = tf.placeholder(tf.float32, [None, 224, 224, 3], name='image_ph')
points_gt = tf.placeholder(tf.float32, [None, args.num_landmarks*2], name='points_gt_ph')
is_training = tf.placeholder(tf.bool, name='is_training')
# construct loss
inference, _ = AlexNet.alexnet_v2(images, args.num_landmarks*2, is_training, args.dropout_keep_prob)
loss = tf.reduce_mean(NormRmse(GroudTruth=points_gt, Prediction=inference, num_points=args.num_landmarks))
# tf.summary.scalar('landmark_loss', loss)
# with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS,'alexnet_v2')):
optimizer = tf.train.AdamOptimizer(0.001).minimize(loss,var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,'alexnet_v2'))
Saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=10)
# merged = tf.summary.merge_all()
Writer = tf.summary.FileWriter(args.log_dir, tf.get_default_graph())
# with tf.Session(config=config) as sess:
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
batch_num = len(image_set) // args.batch_size
step = 0
for epoch in range(args.epoches):
batch_id = 0
while batch_id < args.epoch_size:
RandomIdx = np.random.choice(image_set.shape[0], args.batch_size, False)
print('RandomIdx:', RandomIdx)
start_time = time.time()
def __init__(self, learning_rate, regularizer=None):
self.features = tf.placeholder(tf.float32, [None, 784])
self.labels = tf.placeholder(tf.float32, [None, 10])
self.keep_prob = tf.placeholder(tf.float32)
self.regularizer = regularizer
learning_rate = tf.Variable(learning_rate)
a0 = tf.reshape(self.features, [-1, 28, 28, 1])
# Layer1 (conv+pooling+lrn)
with tf.variable_scope('conv1', reuse=tf.AUTO_REUSE) as scope:
# W:[5,5]是窗口的大小;[1]是输入的厚度;[32]是输出的厚度
W = tf.get_variable(
"W", [5, 5, 1, 32],
initializer=tf.truncated_normal_initializer(stddev=0.1))
if self.regularizer:
tf.add_to_collection('losses', self.regularizer(W))
# b:[32]是输出的厚度
b = tf.get_variable("b", [32],
initializer=tf.constant_initializer(0.1))
# activate:a0是输入的图像们;strides = [1,1,1,1]是步长,一般取这个值就OK了
a = tf.nn.relu(
tf.nn.conv2d(a0, W, strides=[1, 1, 1, 1], padding='SAME') + b,
name='conv2d-relu')
#a = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(a0,W1,strides = [1,1,1,1],padding = 'SAME'), b1))
# pooling:池化操作.就这样子就OK了 = >表示长宽缩小一半而厚度不变.
a_pool = tf.nn.max_pool(a,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pooling')
# lrn层
a1 = tf.nn.lrn(a_pool,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='lrn')
# 最后输出的shape可以通过print(a1.get_shape())查看
# Layer2 (conv+pooling+lrn)
with tf.variable_scope('conv2', reuse=tf.AUTO_REUSE) as scope:
# W:[5,5]是窗口的大小;[1]是输入的厚度;[32]是输出的厚度
W = tf.get_variable(
"W", [5, 5, 32, 64],
initializer=tf.truncated_normal_initializer(stddev=0.1))
if self.regularizer:
tf.add_to_collection('losses', self.regularizer(W))
# b:[32]是输出的厚度
b = tf.get_variable("b", [64],
initializer=tf.constant_initializer(0.1))
# activate:a0是输入的图像们;strides = [1,1,1,1]是步长,一般取这个值就OK了
a = tf.nn.relu(
tf.nn.conv2d(a1, W, strides=[1, 1, 1, 1], padding='SAME') + b,
name='conv2d-relu')
#a = tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(a0,W1,strides = [1,1,1,1],padding = 'SAME'), b1))
# pooling:池化操作.就这样子就OK了 = >表示长宽缩小一半而厚度不变.
a_pool = tf.nn.max_pool(a,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID',
name='pooling')
# lrn层
a2 = tf.nn.lrn(a_pool,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='lrn')
# 最后输出的shape可以通过print(a_norm.get_shape())查看
with tf.name_scope('transition'):
#tranisition from conv to fully_connected
a2_tran = tf.reshape(a2, [-1, 7 * 7 * 64])
# Layer3 (fully_connected)
a3 = self.nn(a2_tran,
1024,
tf.nn.sigmoid,
regularizer=self.regularizer,
scope_name='fully1')
# dropout
a3_dropout = tf.nn.dropout(a3, self.keep_prob)
# Layer4 (fully_connected)
y_predicted = self.nn(a3_dropout,
10,
tf.nn.softmax,
regularizer=self.regularizer,
scope_name='fully2')
with tf.name_scope('loss'):
# loss
cross_entropy = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=self.labels,
logits=y_predicted))
tf.add_to_collection('losses', cross_entropy)
self.loss = tf.add_n(tf.get_collection('losses'))
params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
self.train = self.optimizer(self.loss, params, learning_rate)
with tf.name_scope('accuracy'):
self.total = tf.reduce_sum(
tf.cast(
tf.equal(tf.argmax(y_predicted, axis=1),
tf.argmax(self.labels, axis=1)), tf.int16))
def train():
with tf.Graph().as_default(), tf.device('/cpu:0'):
num_gpu = len(cfgs.GPU_GROUP.strip().split(','))
global_step = slim.get_or_create_global_step()
lr = warmup_lr(cfgs.LR, global_step, cfgs.WARM_SETP, num_gpu)
tf.summary.scalar('lr', lr)
with tf.name_scope('get_batch'):
if cfgs.IMAGE_PYRAMID:
shortside_len_list = tf.constant(cfgs.IMG_SHORT_SIDE_LEN)
shortside_len = tf.random_shuffle(shortside_len_list)[0]
else:
shortside_len = cfgs.IMG_SHORT_SIDE_LEN
img_name_batch, img_batch, gtboxes_and_label_batch, num_objects_batch, img_h_batch, img_w_batch = \
next_batch(dataset_name=cfgs.DATASET_NAME,
batch_size=cfgs.BATCH_SIZE * num_gpu,
shortside_len=shortside_len,
is_training=True)
optimizer = tf.train.MomentumOptimizer(lr, momentum=cfgs.MOMENTUM)
r3det = build_whole_network_r3det.DetectionNetwork(
base_network_name=cfgs.NET_NAME,
is_training=True)
# data processing
inputs_list = []
for i in range(num_gpu):
img = tf.expand_dims(img_batch[i], axis=0)
if cfgs.NET_NAME in ['resnet152_v1d', 'resnet101_v1d', 'resnet50_v1d']:
img = img / tf.constant([cfgs.PIXEL_STD])
gtboxes_and_label_r = tf.py_func(backward_convert,
inp=[gtboxes_and_label_batch[i]],
Tout=tf.float32)
gtboxes_and_label_r = tf.reshape(gtboxes_and_label_r, [-1, 6])
gtboxes_and_label_h = get_horizen_minAreaRectangle(gtboxes_and_label_batch[i])
gtboxes_and_label_h = tf.reshape(gtboxes_and_label_h, [-1, 5])
num_objects = num_objects_batch[i]
num_objects = tf.cast(tf.reshape(num_objects, [-1, ]), tf.float32)
img_h = img_h_batch[i]
img_w = img_w_batch[i]
inputs_list.append([img, gtboxes_and_label_h, gtboxes_and_label_r, num_objects, img_h, img_w])
tower_grads = []
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(cfgs.WEIGHT_DECAY)
total_loss_dict = {
'cls_loss': tf.constant(0., tf.float32),
'reg_loss': tf.constant(0., tf.float32),
'refine_cls_loss': tf.constant(0., tf.float32),
'refine_reg_loss': tf.constant(0., tf.float32),
'refine_cls_loss_stage3': tf.constant(0., tf.float32),
'refine_reg_loss_stage3': tf.constant(0., tf.float32),
'total_losses': tf.constant(0., tf.float32),
}
if cfgs.USE_SUPERVISED_MASK:
total_loss_dict['mask_loss'] = tf.constant(0., tf.float32)
with tf.variable_scope(tf.get_variable_scope()):
for i in range(num_gpu):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i):
with slim.arg_scope(
[slim.model_variable, slim.variable],
device='/device:CPU:0'):
with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane,
slim.conv2d_transpose, slim.separable_conv2d, slim.fully_connected],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
biases_initializer=tf.constant_initializer(0.0)):
gtboxes_and_label_h, gtboxes_and_label_r = tf.py_func(get_gtboxes_and_label,
inp=[inputs_list[i][1],
inputs_list[i][2],
inputs_list[i][3]],
Tout=[tf.float32, tf.float32])
gtboxes_and_label_h = tf.reshape(gtboxes_and_label_h, [-1, 5])
gtboxes_and_label_r = tf.reshape(gtboxes_and_label_r, [-1, 6])
img = inputs_list[i][0]
img_shape = inputs_list[i][-2:]
img = tf.image.crop_to_bounding_box(image=img,
offset_height=0,
offset_width=0,
target_height=tf.cast(img_shape[0], tf.int32),
target_width=tf.cast(img_shape[1], tf.int32))
outputs = r3det.build_whole_detection_network(input_img_batch=img,
gtboxes_batch_h=gtboxes_and_label_h,
gtboxes_batch_r=gtboxes_and_label_r,
gpu_id=i)
gtboxes_in_img_h = draw_boxes_with_categories(img_batch=img,
boxes=gtboxes_and_label_h[:, :-1],
labels=gtboxes_and_label_h[:, -1],
method=0)
gtboxes_in_img_r = draw_boxes_with_categories(img_batch=img,
boxes=gtboxes_and_label_r[:, :-1],
labels=gtboxes_and_label_r[:, -1],
method=1)
tf.summary.image('Compare/gtboxes_h_gpu:%d' % i, gtboxes_in_img_h)
tf.summary.image('Compare/gtboxes_r_gpu:%d' % i, gtboxes_in_img_r)
if cfgs.ADD_BOX_IN_TENSORBOARD:
detections_in_img = draw_boxes_with_categories_and_scores(
img_batch=img,
boxes=outputs[0],
scores=outputs[1],
labels=outputs[2],
method=1)
tf.summary.image('Compare/final_detection_gpu:%d' % i, detections_in_img)
loss_dict = outputs[-1]
total_losses = 0.0
for k in loss_dict.keys():
total_losses += loss_dict[k]
total_loss_dict[k] += loss_dict[k] / num_gpu
total_losses = total_losses / num_gpu
total_loss_dict['total_losses'] += total_losses
if i == num_gpu - 1:
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
# weight_decay_loss = tf.add_n(slim.losses.get_regularization_losses())
total_losses = total_losses + tf.add_n(regularization_losses)
tf.get_variable_scope().reuse_variables()
grads = optimizer.compute_gradients(total_losses)
if cfgs.GRADIENT_CLIPPING_BY_NORM is not None:
grads = slim.learning.clip_gradient_norms(grads, cfgs.GRADIENT_CLIPPING_BY_NORM)
tower_grads.append(grads)
for k in total_loss_dict.keys():
tf.summary.scalar('{}/{}'.format(k.split('_')[0], k), total_loss_dict[k])
if len(tower_grads) > 1:
grads = sum_gradients(tower_grads)
else:
grads = tower_grads[0]
if cfgs.MUTILPY_BIAS_GRADIENT is not None:
final_gvs = []
with tf.variable_scope('Gradient_Mult'):
for grad, var in grads:
scale = 1.
if '/biases:' in var.name:
scale *= cfgs.MUTILPY_BIAS_GRADIENT
if 'conv_new' in var.name:
scale *= 3.
if not np.allclose(scale, 1.0):
grad = tf.multiply(grad, scale)
final_gvs.append((grad, var))
apply_gradient_op = optimizer.apply_gradients(final_gvs, global_step=global_step)
else:
apply_gradient_op = optimizer.apply_gradients(grads, global_step=global_step)
variable_averages = tf.train.ExponentialMovingAverage(0.9999, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
train_op = tf.group(apply_gradient_op, variables_averages_op)
# train_op = optimizer.apply_gradients(final_gvs, global_step=global_step)
summary_op = tf.summary.merge_all()
restorer, restore_ckpt = r3det.get_restorer()
saver = tf.train.Saver(max_to_keep=5)
init_op = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer()
)
tfconfig = tf.ConfigProto(
allow_soft_placement=True, log_device_placement=False)
tfconfig.gpu_options.allow_growth = True
with tf.Session(config=tfconfig) as sess:
sess.run(init_op)
# sess.run(tf.initialize_all_variables())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
summary_path = os.path.join(cfgs.SUMMARY_PATH, cfgs.VERSION)
tools.mkdir(summary_path)
summary_writer = tf.summary.FileWriter(summary_path, graph=sess.graph)
if not restorer is None:
restorer.restore(sess, restore_ckpt)
print('restore model')
for step in range(cfgs.MAX_ITERATION // num_gpu):
training_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
if step % cfgs.SHOW_TRAIN_INFO_INTE != 0 and step % cfgs.SMRY_ITER != 0:
_, global_stepnp = sess.run([train_op, global_step])
else:
if step % cfgs.SHOW_TRAIN_INFO_INTE == 0 and step % cfgs.SMRY_ITER != 0:
start = time.time()
_, global_stepnp, total_loss_dict_ = \
sess.run([train_op, global_step, total_loss_dict])
end = time.time()
print('***'*20)
print("""%s: global_step:%d current_step:%d"""
% (training_time, (global_stepnp-1)*num_gpu, step*num_gpu))
print("""per_cost_time:%.3fs"""
% ((end - start) / num_gpu))
loss_str = ''
for k in total_loss_dict_.keys():
loss_str += '%s:%.3f\n' % (k, total_loss_dict_[k])
print(loss_str)
if np.isnan(total_loss_dict_['total_losses']):
sys.exit(0)
else:
if step % cfgs.SMRY_ITER == 0:
_, global_stepnp, summary_str = sess.run([train_op, global_step, summary_op])
summary_writer.add_summary(summary_str, (global_stepnp-1)*num_gpu)
summary_writer.flush()
if (step > 0 and step % (cfgs.SAVE_WEIGHTS_INTE // num_gpu) == 0) or (step >= cfgs.MAX_ITERATION // num_gpu - 1):
save_dir = os.path.join(cfgs.TRAINED_CKPT, cfgs.VERSION)
if not os.path.exists(save_dir):
os.mkdir(save_dir)
save_ckpt = os.path.join(save_dir, '{}_'.format(cfgs.DATASET_NAME) +
str((global_stepnp-1)*num_gpu) + 'model.ckpt')
saver.save(sess, save_ckpt)
print(' weights had been saved')
coord.request_stop()
coord.join(threads)
def run_training():
""" Define our model and train it """
# Create models if its been pretrained, or we're training it
load_generator = a.generator_pretrain is not None
train_generator = a.generator_checkpoints is not None
load_teacher = a.teacher_pretrain is not None
load_student = a.student_pretrain is not None
train_student = a.student_checkpoints is not None
models = {}
with tf.Graph().as_default():
# Define our generator model
if load_generator or train_generator:
with tf.variable_scope('generator'):
#noisy_inputs = tf.placeholder(tf.float32, [None, a.channels, None, a.input_featdim], name='noisy')
noisy_inputs = tf.placeholder(tf.float32, [1, 1, None, a.input_featdim], name='noisy')
output_type = a.loss_weight.keys() & ['fidelity', 'masking', 'map-as-mask-mimic']
if a.generator_model == 'resnet':
generator = ResNet(
inputs = noisy_inputs,
output_dim = a.output_featdim,
output_type = output_type,
fc_nodes = a.gunits,
fc_layers = a.glayers,
filters = a.gfilters,
dropout = a.dropout,
framewise = True,
#addin = True,
)
elif a.generator_model == 'dnn':
from dnn import DNN
generator = DNN(
inputs = noisy_inputs,
output_dim = a.output_featdim,
output_type = output_type,
)
generator_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='generator')
load_generator_vars = [var for var in generator_vars if '_scale' not in var.op.name and '_shift' not in var.op.name]
generator_loader = tf.train.Saver(load_generator_vars)
generator_saver = tf.train.Saver(generator_vars)
models['generator'] = {'model': generator, 'train': train_generator, 'vars': generator_vars}
if load_teacher:
with tf.variable_scope('teacher'):
#clean_inputs = tf.placeholder(tf.float32, [None, a.channels, None, a.output_featdim], name='clean')
clean_inputs = tf.placeholder(tf.float32, [1, 1, None, a.output_featdim], name='clean')
teacher = ResNet(
inputs = clean_inputs,
output_dim = a.senones,
fc_nodes = a.tunits,
fc_layers = a.tlayers,
filters = a.tfilters,
dropout = 0,
framewise = a.framewise_mimic,
#conv_1d = True,
)
teacher_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='teacher')
teacher_saver = tf.train.Saver({'mimic' + var.op.name[7:]: var for var in teacher_vars})
models['teacher'] = {'model': teacher, 'train': False, 'vars': teacher_vars}
# Define critic for generating outputs
if load_student or train_student:
if load_generator or train_generator:
inputs = generator.outputs
else:
#inputs = tf.placeholder(tf.float32, [None, a.channels, None, a.input_featdim], name='clean')
inputs = tf.placeholder(tf.float32, [1, 1, None, a.input_featdim], name='clean')
with tf.variable_scope('mimic'):
if a.student_model == 'resnet':
student = ResNet(
inputs = inputs,
output_dim = a.senones,
fc_nodes = a.sunits,
fc_layers = a.slayers,
filters = a.sfilters,
dropout = a.dropout,
framewise = a.framewise_mimic,
)
elif a.student_model == 'lstm':
from lstm import BiLSTM
student = BiLSTM(
inputs = inputs,
output_shape = [-1, 1, a.characters],
)
student_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='mimic')
student_saver = tf.train.Saver(student_vars)
models['student'] = {'model': student, 'train': train_student, 'vars': student_vars}
flists = []
for flist in [
('clean', 'json', a.clean_flist),
('noisy', 'json', a.noisy_flist),
('noise', 'json', a.noise_flist),
('numpy', 'json', a.numpy_flist),
('clean', 'scp', a.clean_scp),
('noisy', 'scp', a.noisy_scp),
('senone', 'txt', a.senone_file),
('trans', 'txt', a.trans_file),
]:
if flist[-1] is not None:
flists.append(flist)
for loss_type in ['masking', 'map-as-mask-mimic', 'fidelity']:
if loss_type in a.loss_weight and a.loss_weight[loss_type] == 0:
del a.loss_weight[loss_type]
# Create loader for train data
train_loader = DataLoader(
base_dir = a.base_directory,
flists = flists,
stage = 'tr',
shuffle = True,
channels = a.channels,
compute_irm = 'masking' in a.loss_weight,
logify = a.logify,
)
# Create loader
dev_loader = DataLoader(
base_dir = a.base_directory,
flists = flists,
stage = 'dt',
shuffle = False,
channels = a.channels,
compute_irm = 'masking' in a.loss_weight,
logify = a.logify,
)
trainer = Trainer(
models = models,
learn_rate = a.learn_rate,
lr_decay = a.lr_decay,
loss_weight = a.loss_weight,
batch_size = a.batch_size,
)
# Begin session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Load critic weights, generator weights and initialize trainer weights
#if a.generator_pretrain and a.model_file:
# generator_saver.restore(sess, os.path.join(a.generator_pretrain, a.model_file))
if a.generator_pretrain:
sess.run(tf.variables_initializer(generator.scale_vars))
generator_loader.restore(sess, tf.train.latest_checkpoint(a.generator_pretrain))
elif train_generator:
sess.run(tf.variables_initializer(generator_vars))
# Load teacher
if a.teacher_pretrain:
#ckpt = tf.train.latest_checkpoint(a.teacher_pretrain)
#from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file
#print_tensors_in_checkpoint_file(ckpt, all_tensors=False, tensor_name='', all_tensor_names=True)
teacher_saver.restore(sess, tf.train.latest_checkpoint(a.teacher_pretrain))
# Load student
if a.student_pretrain:
if a.student_file:
student_saver.restore(sess, os.path.join(a.student_pretrain, a.student_file))
else:
student_saver.restore(sess, tf.train.latest_checkpoint(a.student_pretrain))
elif train_student:
sess.run(tf.variables_initializer(student_vars))
# Perform training
min_loss = float('inf')
for epoch in range(1, 200):
print('Epoch %d' % epoch)
# Run train ops
losses, duration = trainer.run_ops(sess, train_loader, training = True, epoch = epoch)
for loss in a.loss_weight:
print('{} loss: {:.6f}'.format(loss, losses[loss]))
print('Train loss: %.6f (%.3f sec)' % (losses['average'], duration))
# Run eval ops
losses, duration = trainer.run_ops(sess, dev_loader, training = False)
eval_loss = losses['average']
for loss in a.loss_weight:
print('{} loss: {:.6f}'.format(loss, losses[loss]))
print('Eval loss: %.6f (%.3f sec)\n' % (eval_loss, duration))
if 'cer' in losses:
eval_loss = losses['cer']
# Save if we've got the best loss so far
if eval_loss < min_loss:
min_loss = eval_loss
if a.generator_checkpoints:
save_file = os.path.join(a.generator_checkpoints, "model-{0:.4f}.ckpt".format(eval_loss))
save_path = generator_saver.save(sess, save_file, global_step = epoch)
if a.student_checkpoints:
save_file = os.path.join(a.student_checkpoints, "model-{0:.4f}.ckpt".format(eval_loss))
save_path = student_saver.save(sess, save_file, global_step = epoch)
def inference(reader, train_dir, data_pattern, out_file_location, batch_size,
top_k):
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as sess, gfile.Open(
out_file_location, "w+") as out_file:
video_id_batch, video_batch, num_frames_batch = get_input_data_tensors(
reader, data_pattern, batch_size)
checkpoint_file = os.path.join(FLAGS.train_dir, "inference_model")
if not gfile.Exists(checkpoint_file + ".meta"):
raise IOError("Cannot find %s. Did you run eval.py?" %
checkpoint_file)
meta_graph_location = checkpoint_file + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
if FLAGS.output_model_tgz:
with tarfile.open(FLAGS.output_model_tgz, "w:gz") as tar:
for model_file in glob.glob(checkpoint_file + '.*'):
tar.add(model_file, arcname=os.path.basename(model_file))
tar.add(os.path.join(FLAGS.train_dir, "model_flags.json"),
arcname="model_flags.json")
print('Tarred model onto ' + FLAGS.output_model_tgz)
saver = tf.train.import_meta_graph(meta_graph_location,
clear_devices=True)
logging.info("restoring variables from " + checkpoint_file)
saver.restore(sess, checkpoint_file)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(
set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
out_file.write("VideoId,Labels\n")
try:
while not coord.should_stop():
video_id_batch_val, video_batch_val, num_frames_batch_val = sess.run(
[video_id_batch, video_batch, num_frames_batch])
predictions_val, = sess.run(
[predictions_tensor],
feed_dict={
input_tensor: video_batch_val,
num_frames_tensor: num_frames_batch_val
})
now = time.time()
num_examples_processed += len(video_batch_val)
num_classes = predictions_val.shape[1]
logging.info("num examples processed: " +
str(num_examples_processed) +
" elapsed seconds: " +
"{0:.2f}".format(now - start_time))
for line in format_lines(video_id_batch_val, predictions_val,
top_k):
out_file.write(line)
out_file.flush()
except tf.errors.OutOfRangeError:
logging.info(
'Done with inference. The output file was written to ' +
out_file_location)
finally:
coord.request_stop()
coord.join(threads)
sess.close()
num_inputs=fc_layer_size,
num_outputs=num_classes,
use_relu=False)
y_pred = tf.nn.softmax(layer_fc2,name='y_pred')
y_pred_cls = tf.argmax(y_pred, dimension=1)
session.run(tf.global_variables_initializer())
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=layer_fc2,
labels=y_true)
cost = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)
correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='conv1')
tf.get_variable_scope().reuse_variables()
# print(tf.trainable_variables())
weights = vars[0]
#TO-DO: Normalize by channel and image
#x_min = tf.reduce_min(weights)
#x_max = tf.reduce_max(weights)
#weights_0_to_1 = (weights - x_min) / (x_max - x_min)
#weights_0_to_255_uint8 = tf.image.convert_image_dtype (weights_0_to_1, dtype=tf.uint8)
weights_transposed = tf.transpose (weights, [3, 0, 1, 2])
#print(weights)
#?summary_writer = tf.summary.FileWriter('summary_dir');
#summary_writer.add_graph(graph=tf.get_default_graph())
session.run(tf.global_variables_initializer())
def _vars(self, scope):
res = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.scope + '/' + scope)
assert len(res) > 0
return res
def main(_):
# word_id_mapping_o, w2v_o = load_w2v(FLAGS.embedding_file, FLAGS.embedding_dim, True)
word_id_mapping_o, w2v_o = load_word_embedding(FLAGS.word_id_file, FLAGS.embedding_file, FLAGS.embedding_dim, True)
word_embedding_o = tf.constant(w2v_o, dtype=tf.float32)
# word_id_mapping_r, w2v_r = load_w2v(FLAGS.embedding_file_r, FLAGS.embedding_dim, True)
# word_id_mapping_r, w2v_r = load_word_embedding(FLAGS.word_id_file, FLAGS.embedding_file_r, FLAGS.embedding_dim, True)
word_id_mapping_r = word_id_mapping_o
word_embedding_r = tf.constant(w2v_o, dtype=tf.float32)
with tf.name_scope('inputs'):
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
x_o = tf.placeholder(tf.int32, [None, FLAGS.max_doc_len, FLAGS.max_sentence_len])
x_r = tf.placeholder(tf.int32, [None, FLAGS.max_doc_len, FLAGS.max_sentence_len])
sen_len_o = tf.placeholder(tf.int32, [None, FLAGS.max_doc_len])
sen_len_r = tf.placeholder(tf.int32, [None, FLAGS.max_doc_len])
doc_len_o = tf.placeholder(tf.int32, None)
doc_len_r = tf.placeholder(tf.int32, None)
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
inputs_o = tf.nn.embedding_lookup(word_embedding_o, x_o)
inputs_o = tf.reshape(inputs_o, [-1, FLAGS.max_sentence_len, FLAGS.embedding_dim])
inputs_r = tf.nn.embedding_lookup(word_embedding_r, x_r)
inputs_r = tf.reshape(inputs_r, [-1, FLAGS.max_sentence_len, FLAGS.embedding_dim])
prob = hn_inter_att(inputs_o, sen_len_o, doc_len_o, inputs_r, sen_len_r, doc_len_r, keep_prob1, keep_prob2)
with tf.name_scope('loss'):
reg_loss = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prob, labels=y)) + tf.add_n(reg_loss)
all_vars = [var for var in tf.global_variables()]
with tf.name_scope('train'):
global_step = tf.Variable(0, name='global_step', trainable=False)
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
grads, global_norm = tf.clip_by_global_norm(tf.gradients(loss, all_vars), 5.0)
train_op = optimizer.apply_gradients(zip(grads, all_vars), name='train_op', global_step=global_step)
with tf.name_scope('predict'):
cor_pred = tf.equal(tf.argmax(prob, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(cor_pred, tf.float32))
accuracy_num = tf.reduce_sum(tf.cast(cor_pred, tf.int32))
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1,
FLAGS.keep_prob2,
FLAGS.batch_size,
FLAGS.learning_rate,
FLAGS.l2_reg,
FLAGS.max_sentence_len,
FLAGS.embedding_dim,
FLAGS.n_hidden,
FLAGS.n_class
)
def get_batch_data(xo, slo, dlo, xr, slr, dlr, yy, batch_size, kp1, kp2, is_shuffle=True):
for index in batch_index(len(yy), batch_size, 1, is_shuffle):
feed_dict = {
x_o: xo[index],
x_r: xr[index],
y: yy[index],
sen_len_o: slo[index],
sen_len_r: slr[index],
doc_len_o: dlo[index],
doc_len_r: dlr[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
conf = tf.ConfigProto(allow_soft_placement=True)
conf.gpu_options.allow_growth = True
with tf.Session(config=conf) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, accuracy, test_loss, test_acc, _dir, title, sess)
save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
saver = saver_func(save_dir)
init = tf.global_variables_initializer()
sess.run(init)
# saver.restore(sess, '/-')
tr_x, tr_y, tr_sen_len, tr_doc_len = load_inputs_document(
FLAGS.train_file,
word_id_mapping_o,
FLAGS.max_sentence_len,
FLAGS.max_doc_len
)
te_x, te_y, te_sen_len, te_doc_len = load_inputs_document(
FLAGS.test_file,
word_id_mapping_o,
FLAGS.max_sentence_len,
FLAGS.max_doc_len
)
tr_x_r, tr_y_r, tr_sen_len_r, tr_doc_len_r = load_inputs_document(
FLAGS.train_file_r,
word_id_mapping_r,
FLAGS.max_sentence_len,
FLAGS.max_doc_len
)
te_x_r, te_y_r, te_sen_len_r, te_doc_len_r = load_inputs_document(
FLAGS.test_file_r,
word_id_mapping_r,
FLAGS.max_sentence_len,
FLAGS.max_doc_len
)
# v_x, v_y, v_sen_len, v_doc_len = load_inputs_document(
# FLAGS.validate_file_path,
# word_id_mapping,
# FLAGS.max_sentence_len,
# FLAGS.max_doc_len
# )
# v_x, v_y, v_sen_len, v_doc_len = load_inputs_document(
# FLAGS.validate_file_path,
# word_id_mapping,
# FLAGS.max_sentence_len,
# FLAGS.max_doc_len
# )
max_acc, max_prob, step = 0., None, None
max_ty, max_py = None, None
for i in xrange(FLAGS.n_iter):
for train, _ in get_batch_data(tr_x, tr_sen_len, tr_doc_len, tr_x_r, tr_sen_len_r, tr_doc_len_r, tr_y,
FLAGS.batch_size, FLAGS.keep_prob1, FLAGS.keep_prob2):
_, step, summary = sess.run([train_op, global_step, train_summary_op], feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat([tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]), 0)
# sess.run(embed_update)
acc, cost, cnt = 0., 0., 0
p, ty, py = [], [], []
for test, num in get_batch_data(te_x, te_sen_len, te_doc_len, te_x_r, te_sen_len_r, te_doc_len_r, te_y, FLAGS.batch_size, 1.0, 1.0, False):
_loss, _acc, _p, _ty, _py = sess.run([loss, accuracy_num, prob, true_y, pred_y], feed_dict=test)
p += list(_p)
ty += list(_ty)
py += list(_py)
acc += _acc
cost += _loss * num
cnt += num
print 'all samples={}, correct prediction={}'.format(cnt, acc)
acc = acc / cnt
cost = cost / cnt
print 'Iter {}: mini-batch loss={:.6f}, test acc={:.6f}'.format(i, cost, acc)
summary = sess.run(test_summary_op, feed_dict={test_loss: cost, test_acc: acc})
test_summary_writer.add_summary(summary, step)
if acc > max_acc:
max_acc = acc
max_prob = p
max_ty = ty
max_py = py
# saver.save(sess, save_dir, global_step=step)
print 'P:', precision_score(max_ty, max_py, average=None)
print 'R:', recall_score(max_ty, max_py, average=None)
print 'F:', f1_score(max_ty, max_py, average=None)
fp = open(FLAGS.prob_file, 'w')
for item in max_prob:
fp.write(' '.join([str(it) for it in item]) + '\n')
print 'Optimization Finished! Max acc={}'.format(max_acc)
print 'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'.format(
FLAGS.learning_rate,
FLAGS.n_iter,
FLAGS.batch_size,
FLAGS.n_hidden,
FLAGS.l2_reg
)
def connect_data_and_network(self,
outputs_collector=None,
gradients_collector=None):
def switch_sampler(for_training):
with tf.name_scope('train' if for_training else 'validation'):
sampler = self.get_sampler()[0][0 if for_training else -1]
return sampler.pop_batch_op()
if self.is_training:
if self.action_param.validation_every_n > 0:
data_dict = tf.cond(tf.logical_not(self.is_validation),
lambda: switch_sampler(True),
lambda: switch_sampler(False))
else:
data_dict = switch_sampler(for_training=True)
image = tf.cast(data_dict['image'], tf.float32)
net_output = self.net(image, is_training=self.is_training)
with tf.name_scope('Optimiser'):
optimiser_class = OptimiserFactory.create(
name=self.action_param.optimiser)
self.optimiser = optimiser_class.get_instance(
learning_rate=self.action_param.lr)
loss_func = LossFunction(loss_type=self.action_param.loss_type)
data_loss = loss_func(net_output)
loss = data_loss
if self.net_param.decay > 0.0:
reg_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
if reg_losses:
reg_loss = tf.reduce_mean(
[tf.reduce_mean(reg_loss) for reg_loss in reg_losses])
loss = loss + reg_loss
grads = self.optimiser.compute_gradients(loss)
# collecting gradients variables
gradients_collector.add_to_collection([grads])
outputs_collector.add_to_collection(var=data_loss,
name='variational_lower_bound',
average_over_devices=True,
collection=CONSOLE)
outputs_collector.add_to_collection(var=data_loss,
name='variational_lower_bound',
average_over_devices=True,
summary_type='scalar',
collection=TF_SUMMARIES)
outputs_collector.add_to_collection(var=net_output[4],
name='Originals',
average_over_devices=False,
summary_type='image3_coronal',
collection=TF_SUMMARIES)
outputs_collector.add_to_collection(var=net_output[2],
name='Means',
average_over_devices=False,
summary_type='image3_coronal',
collection=TF_SUMMARIES)
outputs_collector.add_to_collection(var=net_output[5],
name='Variances',
average_over_devices=False,
summary_type='image3_coronal',
collection=TF_SUMMARIES)
else:
if self._infer_type in ('encode', 'encode-decode'):
data_dict = self.get_sampler()[0][0].pop_batch_op()
image = tf.cast(data_dict['image'], dtype=tf.float32)
net_output = self.net(image, is_training=False)
outputs_collector.add_to_collection(
var=data_dict['image_location'],
name='location',
average_over_devices=True,
collection=NETWORK_OUTPUT)
if self._infer_type == 'encode-decode':
outputs_collector.add_to_collection(
var=net_output[2],
name='generated_image',
average_over_devices=True,
collection=NETWORK_OUTPUT)
if self._infer_type == 'encode':
outputs_collector.add_to_collection(
var=net_output[7],
name='embedded',
average_over_devices=True,
collection=NETWORK_OUTPUT)
self.output_decoder = WindowAsImageAggregator(
image_reader=self.readers[0],
output_path=self.action_param.save_seg_dir)
return
elif self._infer_type == 'sample':
image_size = (self.net_param.batch_size,) + \
self.action_param.spatial_window_size + (1,)
dummy_image = tf.zeros(image_size)
net_output = self.net(dummy_image, is_training=False)
noise_shape = net_output[-1].shape.as_list()
stddev = self.autoencoder_param.noise_stddev
noise = tf.random_normal(shape=noise_shape,
mean=0.0,
stddev=stddev,
dtype=tf.float32)
partially_decoded_sample = self.net.shared_decoder(
noise, is_training=False)
decoder_output = self.net.decoder_means(
partially_decoded_sample, is_training=False)
outputs_collector.add_to_collection(var=decoder_output,
name='generated_image',
average_over_devices=True,
collection=NETWORK_OUTPUT)
self.output_decoder = WindowAsImageAggregator(
image_reader=None,
output_path=self.action_param.save_seg_dir)
return
elif self._infer_type == 'linear_interpolation':
# construct the entire network
image_size = (self.net_param.batch_size,) + \
self.action_param.spatial_window_size + (1,)
dummy_image = tf.zeros(image_size)
net_output = self.net(dummy_image, is_training=False)
data_dict = self.get_sampler()[0][0].pop_batch_op()
real_code = data_dict['feature']
real_code = tf.reshape(real_code, net_output[-1].get_shape())
partially_decoded_sample = self.net.shared_decoder(
real_code, is_training=False)
decoder_output = self.net.decoder_means(
partially_decoded_sample, is_training=False)
outputs_collector.add_to_collection(var=decoder_output,
name='generated_image',
average_over_devices=True,
collection=NETWORK_OUTPUT)
outputs_collector.add_to_collection(
var=data_dict['feature_location'],
name='location',
average_over_devices=True,
collection=NETWORK_OUTPUT)
self.output_decoder = WindowAsImageAggregator(
image_reader=self.readers[0],
output_path=self.action_param.save_seg_dir)
else:
raise NotImplementedError
def _global_vars(self, scope):
res = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.scope + '/' + scope)
return res
def make_scaffold(stage_id, optimizer_var_list, **kwargs):
"""Makes a custom scaffold.
The scaffold
- restores variables from the last training stage.
- initializes new variables in the new block.
Args:
stage_id: An integer of stage id.
optimizer_var_list: A list of optimizer variables.
**kwargs: A dictionary of
'train_root_dir': A string of root directory of training logs.
'num_resolutions': An integer of number of progressive resolutions.
'stable_stage_num_images': An integer of number of training images in
the stable stage.
'transition_stage_num_images': An integer of number of training images
in the transition stage.
'total_num_images': An integer of total number of training images.
Returns:
A `Scaffold` object.
"""
# Holds variables that from the previous stage and need to be restored.
restore_var_list = []
prev_ckpt = None
curr_ckpt = tf.train.latest_checkpoint(make_train_sub_dir(stage_id, **kwargs))
if stage_id > 0 and curr_ckpt is None:
prev_ckpt = tf.train.latest_checkpoint(
make_train_sub_dir(stage_id - 1, **kwargs))
num_blocks, _ = get_stage_info(stage_id, **kwargs)
prev_num_blocks, _ = get_stage_info(stage_id - 1, **kwargs)
# Holds variables created in the new block of the current stage. If the
# current stage is a stable stage (except the initial stage), this list
# will be empty.
new_block_var_list = []
for block_id in range(prev_num_blocks + 1, num_blocks + 1):
new_block_var_list.extend(
tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES,
scope='.*/{}/'.format(networks.block_name(block_id))))
# Every variables that are 1) not for optimizers and 2) from the new block
# need to be restored.
restore_var_list = [
var for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
if var not in set(optimizer_var_list + new_block_var_list)
]
# Add saver op to graph. This saver is used to restore variables from the
# previous stage.
saver_for_restore = tf.train.Saver(
var_list=restore_var_list, allow_empty=True)
# Add the op to graph that initializes all global variables.
init_op = tf.global_variables_initializer()
def _init_fn(unused_scaffold, sess):
# First initialize every variables.
sess.run(init_op)
logging.info('\n'.join([var.name for var in restore_var_list]))
# Then overwrite variables saved in previous stage.
if prev_ckpt is not None:
saver_for_restore.restore(sess, prev_ckpt)
# Use a dummy init_op here as all initialization is done in init_fn.
return tf.train.Scaffold(init_op=tf.constant([]), init_fn=_init_fn)
def main():
images = MNIST # Feed your data here! The program expects batches of 128x128x3 float32 (normalized to be between 0 and 1) images by default
tf.image_summary("real", images, max_images=1)
z = tf.placeholder(tf.float32, [batch_size, z_dim], name='z')
with tf.variable_scope("generator") as scope:
gen = generator(z)
tf.image_summary("fake", gen, max_images=1)
with tf.variable_scope("discriminator") as scope:
disc_real = discriminator(images)
scope.reuse_variables()
disc_fake = discriminator(gen)
# Define Losses
disc_real_loss = losses.sigmoid_cross_entropy(disc_real,
tf.ones([batch_size, 1]))
disc_fake_loss = losses.sigmoid_cross_entropy(
disc_fake, tf.fill([batch_size, 1], -1.0))
d_loss = disc_real_loss + disc_fake_loss
g_loss = losses.sigmoid_cross_entropy(disc_fake, tf.ones([batch_size, 1]))
tf.scalar_summary("Discriminator_loss_real", disc_real_loss)
tf.scalar_summary("Discrimintator_loss_fake", disc_fake_loss)
tf.scalar_summary("Discriminator_loss", d_loss)
tf.scalar_summary("Generator_loss", g_loss)
# The paper found RMSProp to work better than Adam or other momentum based methods
d_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
g_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
d_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
"discriminator")
g_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator")
# Create training ops
d_train_op = slim.learning.create_train_op(d_loss,
d_optimizer,
variables_to_train=d_vars)
g_train_op = slim.learning.create_train_op(g_loss,
g_optimizer,
variables_to_train=g_vars)
# Create clipping ops, thanks to PatrykChrabaszcz for this!
clip_discriminator = []
for var in d_vars:
clip_discriminator.append(tf.assign(var, tf.clip_by_value(var, -c, c)))
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter(log_dir, sess.graph)
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
sess.run(init_op)
start = 0
# If a checkpoint is found, restore what you can. If not, continue
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Checkpoint found! Restoring...")
saver.restore(sess, ckpt.model_checkpoint_path)
# Hackey way to determine what step we're starting on. It feels like there should be some in built function in TensorFlow to do this but I can't find any...
start = int(ckpt.model_checkpoint_path.split("-")[-1]) + 1
print("Restored!")
else:
print("No checkpoint found!")
try:
current_step = start
print("Starting training!")
for itr in xrange(start, max_iterations):
# As per the reference implementation, the discriminator gets a lot of training early on
if current_step < 25 or current_step % 500 == 0:
diters = 100
else:
diters = d_iters
# Train discriminator several times
for i in xrange(diters):
# Clip all discriminator weights to be between -c and c
if i % clip_per == 0:
sess.run(clip_discriminator)
batch_z = np.random.uniform(
-1, 1, [batch_size, z_dim]).astype(np.float32)
sess.run(d_train_op, feed_dict={z: batch_z})
# Train generator once
batch_z = np.random.uniform(-1, 1, [batch_size, z_dim]).astype(
np.float32)
sess.run(g_train_op, feed_dict={z: batch_z})
# Give the user some feedback
if itr % sum_per == 0:
g_loss_val, d_loss_val, summary_str = sess.run(
[g_loss, d_loss, summary_op], feed_dict={z: batch_z})
print(
"Step: %d, Generator Loss: %g, Discriminator Loss: %g"
% (itr, g_loss_val, d_loss_val))
summary_writer.add_summary(summary_str, itr)
# Every so often save
if itr % save_per == 0:
saver.save(sess,
os.path.join(log_dir, "model.ckpt"),
global_step=itr)
current_step = itr
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached!')
except KeyboardInterrupt:
print("Ending training...")
# User terminated with Ctrl-C, save current state
saver.save(sess,
os.path.join(log_dir, "model.ckpt"),
global_step=current_step)
finally:
coord.request_stop()
coord.join(threads)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
##logging tools
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Create global_step.
with tf.device('/cpu:0'):
global_step = tf.train.create_global_step()
# Select the dataset.
dataset = FLAGS.dataset_dir + '%s_%s.tfrecord' %(FLAGS.dataset_name,FLAGS.dataset_split_name)
#image = tf.placeholder(tf.float32, shape=[62,62])
#label = tf.placeholder(tf.float32, shape=[62,62])
with tf.device('/cpu:0'):
with tf.name_scope('input'):
filename_queue = tf.train.string_input_producer([dataset], num_epochs=FLAGS.num_epochs)
image,mask_class,_ = tf_utils.read_and_decode_for_lstm(filename_queue, batch_size = FLAGS.batch_size,\
capacity=20 * FLAGS.batch_size,\
num_threads=FLAGS.num_readers,\
min_after_dequeue=10 * FLAGS.batch_size, is_training=True)
#image = tf.image.resize_images(image,[256,256])
#labels = tf.expand_dims(mask_class, axis=-1)
#labels = tf.image.resize_nearest_neighbor(labels,[256,256])
# reshape mask_class as [-1, num_classes]
labels = tf.to_float(tf.contrib.layers.one_hot_encoding(mask_class,FLAGS.num_classes))
mask_class_onehot_for_d = tf.to_float(tf.contrib.layers.one_hot_encoding(mask_class,FLAGS.num_classes, on_value=0.99,off_value=0.01))
labels = tf.reshape(labels, (-1, FLAGS.num_classes))
#input_d = tf.reshape(mask_class_onehot_for_d, (-1, FLAGS.num_classes))
# onedim_class = tf.reshape(mask_class, (-1,))
#image = preprocessing.data_augmentation(image, is_training=True)
#image,mask_class,mask_instance = preprocessing(image,mask_class,mask_instance,is_train=True,data_format= 'NHWC')
logits,_,_= SpinePathNet.g_l_net(image, batch_size=FLAGS.batch_size, class_num=FLAGS.num_classes, reuse=False, is_training=True, scope='g_SpinePathNet')
#Gan simultanously descriminate the input is the segmentation predictions or ground truth.
D_logit_real = SpinePathNet.d_net(mask_class_onehot_for_d, class_num = FLAGS.num_classes, reuse=None, is_training=True, scope='d_gan')
D_logit_fake = SpinePathNet.d_net(logits, class_num = FLAGS.num_classes, reuse=True, is_training=True, scope='d_gan')
with tf.name_scope('cross_entropy_loss'):
cross_entropy_loss = losses.weighted_cross_entropy_with_logits(FLAGS, Fold, logits, labels)
with tf.name_scope('gan_loss'): # GAN.
D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))
#Accor
D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))
G_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))
#lmada = tf.Variable(2., name = 'weight_of_gan')
D_loss = D_loss_real + D_loss_fake
G_loss = cross_entropy_loss + G_loss_fake
# trainable varibles of generative and discrimative models.
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]
# print (g_vars)
learning_rate = tf_utils.configure_learning_rate(FLAGS,FLAGS.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
optimizer_gan = tf.train.AdamOptimizer(beta1=0.5, learning_rate=0.001)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
#Note: when training, the moving_mean and moving_variance need to be updated.
with tf.control_dependencies(update_ops):
train_op_G = optimizer.minimize(G_loss, global_step=global_step, var_list=g_vars)
#train_op_fake = optimizer.minimize(G_loss_fake, var_list=g_vars)
train_op_D = optimizer_gan.minimize(D_loss, global_step=global_step, var_list=d_vars)
# The op for initializing the variables.
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# Create a session for running operations in the Graph.
#config = tf.ConfigProto(allow_soft_placement = True)
sess = tf.Session()
# Initialize the variables (the trained variables and the
# epoch counter).
sess.run(init_op)
#Include max_to_keep=5
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
exclude_list = FLAGS.ignore_missing_vars
variables_list = tf.contrib.framework.get_variables_to_restore(exclude=exclude_list)
restore = tf.train.Saver(variables_list)
restore.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
global_step = int(global_step)
else: global_step = 0
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Save models.
if not tf.gfile.Exists(FLAGS.train_dir):
#tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
try:
step = global_step # TODO: Continue to train the model, if the checkpoints are exist.
while not coord.should_stop():
start_time = time.time()
#for i in xrange(50):
_, dl, dlr, dlf = sess.run([train_op_D, D_loss, D_loss_real, D_loss_fake])
#cl = sess.run(cross_entropy_loss)
#print('Step %d: cross entropy loss = %.2f, real discrimator loss = %.2f, fake discrimator loss = %.2f' % (step, cl, dlr,dlf))
#step += 50
#for i in xrange(50):
#
_,gl, cel, fake_loss, lr = sess.run([train_op_G, G_loss, cross_entropy_loss, G_loss_fake, learning_rate])
duration = time.time() - start_time
if step % 10 == 0:
print('Step %d: All generative loss = %.2f (Cross entropy loss = %.2f, Fake loss of generatation = %.2f); All discrimator loss = %.2f (Discrimator loss of real = %.2f, Discrimator loss of fake = %.2f); Learning rate = %.4f (%.3f sec)' % (step, gl, cel, fake_loss, dl, dlr, dlf, lr, duration))
step += 1
if step % 1000 == 0: #or (step + 1) == FLAGS.max_steps
#Increase Gan_loss.
#lmada.assign_add(0.1)
saver.save(sess, checkpoint_path, global_step=step)
##Add the model evaluatation in the future.
except tf.errors.OutOfRangeError:
print('Done training for %d epochs, %d steps.' % (FLAGS.num_epochs, step))
saver.save(sess, checkpoint_path, global_step=step)
##Add the model evaluatation in the future.
print('Model is saved as %s') % checkpoint_path
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
sess.close()
# Calculate the loss for one tower of the CIFAR model. This function
# constructs the entire CIFAR model but shares the variables across
# all towers.
start = i * batch_size_gpu
end = start + batch_size_gpu
logits = model(images[start:end, :])
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(
labels=labels[start:end, :], logits=logits),
name="softmax_cross_entropy")
# Reuse variables for the next tower.
tf.get_variable_scope().reuse_variables()
# Retain the summaries from the final tower.
summaries = tf.get_collection(tf.GraphKeys.SUMMARIES,
scope)
# Calculate the gradients for the batch of data on this CIFAR tower.
grads_gpu = opt.compute_gradients(loss)
# Keep track of the gradients across all towers.
tower_grads.append(grads_gpu)
y_pred_classes.append(
tf.argmax(tf.nn.softmax(logits), axis=1))
losses.append(loss)
# We must calculate the mean of each gradient. Note that this is the
# synchronization point across all towers.
grads = average_gradients(tower_grads)
#summaries.append(tf.summary.scalar('learning_rate', lr))
def train_saved_model(config,
checkpoint_file,
x_train,
y_train,
x_dev,
y_dev,
k_fold_id=None,
verbose=0):
"""
Load saved model from checkpoint file and continue training.
Allows changed learning rate, rest stays the same.
:param config:
:param checkpoint_file:
:param x_train:
:param y_train:
:param x_dev:
:param y_dev:
:param verbose:
:return:
"""
write_summaries = True
if config["cross_valid"] != 0:
# Turn off summaries for cross validation as they are not working then
write_summaries = False
if verbose >= 2:
print("Turned off summaries for cross validation to avoid errors.")
sess = tf.Session()
if k_fold_id is not None:
variable_scope = "k_fold_" + str(k_fold_id)
else:
variable_scope = ""
tf.get_variable_scope().reuse_variables()
# Load model
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
if verbose >= 2:
print("Current name scope: {0:s}".format(tf.get_variable_scope().name))
if verbose >= 2:
print("Initializing model: " + config['architecture'] + "...")
if config['architecture'] == "tsainfcnn":
model = TsaInfCnnMapping(sess.graph, variable_scope=variable_scope)
elif config['architecture'] == "attention-cnn":
model = AttentionCNNMapping(sess.graph, variable_scope=variable_scope)
else:
print("ERROR: unrecognized architecture " + config['architecture'])
if verbose >= 1:
print("Model '{0:s}' initialized.".format(config['architecture']))
learning_rate = tf.get_variable("learning_rate",
dtype=tf.float32,
initializer=tf.constant(
[config["learning_rate"]]))
# Update learning rate
lr_assign_op = learning_rate.assign(config["learning_rate"])
sess.run(lr_assign_op)
if k_fold_id is not None:
train_op = tf.get_collection("train_op")[0]
else:
train_op = tf.get_collection("train_op", scope=variable_scope)[0]
# TODO: fix summaries
write_summaries = False
# Setup summaries
if write_summaries:
tf.summary.scalar("loss", model.cross_entropy)
summary_op = tf.summary.merge_all()
# Setup summary writers
out_dir = os.path.join(os.path.abspath(os.path.curdir),
MODEL_OUTPUT_PATH, config['run_name'])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir,
sess.graph)
dev_summary_dir = os.path.join(out_dir, "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir)
best_conf_mat, best_epoch, best_eval_metric = _train_run(
config, sess, model, train_op, saver, x_dev, x_train, y_dev,
y_train, write_summaries, train_summary_writer, summary_op,
dev_summary_writer, verbose)
else:
best_conf_mat, best_epoch, best_eval_metric = _train_run(
config,
sess,
model,
train_op,
saver,
x_dev,
x_train,
y_dev,
y_train,
verbose=verbose)
sess.close()
return best_conf_mat, best_epoch, best_eval_metric, model
def replace_parms(self, sess):
t_parms = tf.get_collection('target_net_params')
e_parms = tf.get_collection('eval_net_params')
sess.run([tf.assign(t, e) for t, e in zip(t_parms, e_parms)])
def variables(self):
assert self.scope is not None, "need to call module once before getting variables"
return tf.get_collection(tf.GraphKeys.VARIABLES, self.scope)
def _make_q_network(self, X_input, name):
'''
Makes the core q network
'''
last = X_input
hidden_initializer = tf.contrib.layers.variance_scaling_initializer()
if self.USE_CONV:
# make convolutional network
conv_num_maps = [32, 64, 64]
# conv_kernel_sizes = [8, 4, 3]
conv_kernel_sizes = [8, 4, 4]
conv_strides = [4, 2, 1]
conv_paddings = ['same'] * 3
conv_activations = [tf.nn.relu] * 3
num_hidden = self.NUM_HIDDEN
elif self.USE_ENCODER:
# experimental autoencoder network
conv_num_maps = [64, 64]
conv_kernel_sizes = [4, 4]
conv_strides = [2, 1]
conv_paddings = ['same'] * len(conv_num_maps)
conv_activations = [tf.nn.relu] * len(conv_num_maps)
num_hidden = self.NUM_HIDDEN
with tf.variable_scope(name) as scope:
if self.USE_CONV or self.USE_ENCODER:
# conv layers
for num_maps, kernel_size, stride, padding, act_func in zip(conv_num_maps,
conv_kernel_sizes,
conv_strides,
conv_paddings,
conv_activations):
last = tf.layers.conv2d(last,
num_maps,
kernel_size=kernel_size,
strides=stride,
padding=padding,
activation=act_func)
input_layer = tf.layers.flatten(last)
if self.conf['use_dueling']:
# action output
last = tf.layers.dense(input_layer,
num_hidden,
activation=tf.nn.relu,
kernel_initializer=hidden_initializer)
advantage = tf.layers.dense(last,
self.num_outputs,
kernel_initializer=hidden_initializer)
# value output
last = tf.layers.dense(input_layer,
num_hidden,
activation=tf.nn.relu,
kernel_initializer=hidden_initializer)
value = tf.layers.dense(last,
1,
kernel_initializer=hidden_initializer)
# combine
outputs = value + tf.subtract(advantage, tf.reduce_mean(advantage,
axis=1,
keepdims=True))
else:
# standard non-dueling architecture
last = tf.layers.dense(input_layer,
num_hidden,
activation=tf.nn.relu,
kernel_initializer=hidden_initializer)
outputs = tf.layers.dense(last,
self.num_outputs,
kernel_initializer=hidden_initializer)
actions = tf.argmax(outputs, axis=1)
# get var names for copy
var_dict = {}
for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=scope.name):
var_dict[var.name[len(scope.name):]] = var
return outputs, actions, var_dict
def backward():
# 输入x的占位符
x = tf.placeholder(tf.float32, [BATCH_SIZE,
forward.IMAGE_SIZE,
forward.IMAGE_SIZE,
forward.NUM_CHANNELS])
# 实际分数y_的占位符
y_ = tf.placeholder(tf.float32, [None, forward.OUTPUT_NODE])
# 得到预测分数y
y = forward.forward(x, True, REGULARIZER)
# 记录训练了多少步
global_step = tf.Variable(0, trainable=False)
# y过一个softmax层，转化为概率，计算y和y_的交叉熵
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
# 求平均
cem = tf.reduce_mean(ce)
# get_collection取出losses集合的值，add_n把值加起来，表示进行正则化
loss = cem + tf.add_n(tf.get_collection('losses'))
# 这一轮训练集的准确率
correct_pred = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# 指数衰减学习率
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
train_num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
# AdamOptimizer：根据损失函数对每个参数的梯度的一阶矩估计和二阶矩估计动态调整针对于每个参数的学习速率
train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss, global_step=global_step)
# 计算w的滑动平均值，记录每个w过去一段时间内的平均值，避免w迅速变化，导致模型过拟合
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
# ema.apply后面的括号是更新列表,每次运行sess.run(ema_op)时,对待训练的参数求滑动平均值
ema_op = ema.apply(tf.trainable_variables())
##将训练过程和计算滑动平均的过程绑定
with tf.control_dependencies([train_op, ema_op]):
# 将它们合并为一个训练节点
train_step = tf.no_op(name='train')
# 实例化一个 tf.train.Saver，之后可以用saver保存模型或读取模型
saver = tf.train.Saver()
# 取BATCH_SIZE数量的训练数据
img_batch, label_batch = generateds.get_tfrecord(BATCH_SIZE, isTrain=True)
with tf.Session() as sess:
# 初始化所有变量
init_op = tf.global_variables_initializer()
sess.run(init_op)
# 断点续训，#如果地址下存在断点，就把断点恢复
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
# 创建线程管理器
coord = tf.train.Coordinator()
# 启动队列填充，读入文件
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(STEPS):
# 运行img_batch和label_batch，获得下一批训练数据
xs, ys = sess.run([img_batch, label_batch])
# 将xs转化为合适的shape准备喂入网络
reshaped_xs = np.reshape(xs, (
BATCH_SIZE,
forward.IMAGE_SIZE,
forward.IMAGE_SIZE,
forward.NUM_CHANNELS))
# 运行之前定义的计算节点，获得输出
_, loss_value, step, acc = sess.run([train_step, loss, global_step, accuracy],
feed_dict={x: reshaped_xs, y_: ys})
# 每10轮保存一次model
if i % 10 == 0:
print("After %d training step(s), loss on training batch is %g. accuracy is %g" % (
step, loss_value, acc))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
# 协调器coord发出所有线程终止信号
coord.request_stop()
# 把开启的线程加入主线程，等待threads结束
coord.join(threads)
def main(_):
best_acc = 0
best_step = 0
best_acc_istrain = 0
best_step_istrain = 0
# We want to see all the logging messages for this tutorial.
tf.logging.set_verbosity(tf.logging.INFO)
# Start a new TensorFlow session.
sess = tf.InteractiveSession()
# Begin by making sure we have the training data we need. If you already have
# training data of your own, use `--data_url= ` on the command line to avoid
# downloading.
model_settings = models.prepare_model_settings(
len(
input_data_filler.prepare_words_list_my(
FLAGS.wanted_words.split(','))), FLAGS.sample_rate,
FLAGS.clip_duration_ms, FLAGS.window_size_ms, FLAGS.window_stride_ms,
FLAGS.dct_coefficient_count)
audio_processor = input_data_filler.AudioProcessor(
FLAGS.data_url, FLAGS.data_dir,
FLAGS.silence_percentage, FLAGS.unknown_percentage,
FLAGS.wanted_words.split(','), FLAGS.validation_percentage,
FLAGS.testing_percentage, model_settings)
fingerprint_size = model_settings['fingerprint_size']
label_count = model_settings['label_count']
time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)
# Figure out the learning rates for each training phase. Since it's often
# effective to have high learning rates at the start of training, followed by
# lower levels towards the end, the number of steps and learning rates can be
# specified as comma-separated lists to define the rate at each stage. For
# example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
# will run 13,000 training loops in total, with a rate of 0.001 for the first
# 10,000, and 0.0001 for the final 3,000.
training_steps_list = list(
map(int, FLAGS.how_many_training_steps.split(',')))
learning_rates_list = list(map(float, FLAGS.learning_rate.split(',')))
if len(training_steps_list) != len(learning_rates_list):
raise Exception(
'--how_many_training_steps and --learning_rate must be equal length '
'lists, but are %d and %d long instead' %
(len(training_steps_list), len(learning_rates_list)))
##############################################
############tensorflow modules##########
fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size],
name='fingerprint_input')
# ############ 模型创建 ##########
istrain = tf.placeholder(tf.bool, name='istrain')
logits = models.create_model(fingerprint_input,
model_settings,
FLAGS.model_architecture,
is_training=istrain)
############ 模型创建 ##########
# logits, dropout_prob= models.create_model(
# fingerprint_input,
# model_settings,
# FLAGS.model_architecture,
# is_training=True)
# Define loss and optimizer
############ 真实值 ##########
ground_truth_input = tf.placeholder(tf.float32, [None, label_count],
name='groundtruth_input')
# Optionally we can add runtime checks to spot when NaNs or other symptoms of
# numerical errors start occurring during training.
control_dependencies = []
if FLAGS.check_nans:
checks = tf.add_check_numerics_ops()
control_dependencies = [checks]
# Create the back propagation and training evaluation machinery in the graph.
############ 交叉熵计算 ##########
# with tf.name_scope('cross_entropy'):
# cross_entropy_mean = tf.reduce_mean(
# tf.nn.softmax_cross_entropy_with_logits(
# labels=ground_truth_input, logits=logits)) + beta*loss_norm
with tf.name_scope('cross_entropy'):
cross_entropy_mean = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=ground_truth_input,
logits=logits))
tf.summary.scalar('cross_entropy', cross_entropy_mean)
############ 学习率、准确率、混淆矩阵 ##########
# learning_rate_input 学习率输入（tf.placeholder）
# train_step 训练过程 （优化器）
# predicted_indices 预测输出索引
# expected_indices 实际希望输出索引
# correct_prediction 正确预测矩阵
# confusion_matrix 混淆矩阵
# evaluation_step 正确分类概率（每个阶段）
# global_step 全局训练阶段
# increment_global_step 全局训练阶段递增
learning_rate_input = tf.placeholder(tf.float32, [],
name='learning_rate_input')
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.AdamOptimizer(learning_rate_input).minimize(
cross_entropy_mean)
# with tf.name_scope('train'), tf.control_dependencies(control_dependencies):
# learning_rate_input = tf.placeholder(
# tf.float32, [], name='learning_rate_input')
# # train_step = tf.train.GradientDescentOptimizer(
# # learning_rate_input).minimize(cross_entropy_mean)
# with tf.control_dependencies(update_ops):
# train_step = tf.train.AdamOptimizer(
# learning_rate_input).minimize(cross_entropy_mean)
predicted_indices = tf.argmax(logits, 1)
expected_indices = tf.argmax(ground_truth_input, 1)
correct_prediction = tf.equal(predicted_indices, expected_indices)
confusion_matrix = tf.confusion_matrix(expected_indices,
predicted_indices,
num_classes=label_count)
evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
acc = tf.summary.scalar('accuracy', evaluation_step)
global_step = tf.train.get_or_create_global_step()
increment_global_step = tf.assign(global_step, global_step + 1)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=None) # max keep file // moren 5
# Merge all the summaries and write them out to /tmp/retrain_logs (by default)
merged_summaries = tf.summary.merge_all()
validation_merged_summaries = tf.summary.merge([
tf.get_collection(tf.GraphKeys.SUMMARIES, 'accuracy'),
tf.get_collection(tf.GraphKeys.SUMMARIES, 'cross_entropy')
])
test_summaries = tf.summary.merge([acc])
test_summaries_istrain = tf.summary.merge([acc])
train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir +
'/validation')
test_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/test')
test_istrain_writer = tf.summary.FileWriter(FLAGS.summaries_dir +
'/test_istrain')
tf.global_variables_initializer().run()
start_step = 1
if FLAGS.start_checkpoint:
models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
start_step = global_step.eval(session=sess)
tf.logging.info('Training from step: %d ', start_step)
# Save graph.pbtxt.
tf.train.write_graph(sess.graph_def, FLAGS.train_dir,
FLAGS.model_architecture + '.pbtxt')
# Save list of words.
with gfile.GFile(
os.path.join(FLAGS.train_dir,
FLAGS.model_architecture + '_labels.txt'), 'w') as f:
f.write('\n'.join(audio_processor.words_list))
###
# model1: fc
# model2: conv :940k个parameter
# model3:low_latancy_conv:~~model1
# model4: 750k
# Training loop.
#############################################
######## 主循环 ######
#############################################
training_steps_max = np.sum(training_steps_list)
for training_step in xrange(start_step, training_steps_max + 1):
# Figure out what the current learning rate is.
####### 自动切换学习率 #######
if training_step < 12000 + 1:
learning_rate_value = learning_rates_list[0] * 0.02**(
training_step / 12000)
else:
learning_rate_value = learning_rates_list[0] * 0.02 #0.015 12000
training_steps_sum = 0
# for i in range(len(training_steps_list)):
# training_steps_sum += training_steps_list[i]
# if training_step <= training_steps_sum:
# learning_rate_value = learning_rates_list[i]
# break
# Pull the audio samples we'll use for training.
####### audio处理器导入数据 ##################################
##get_data(self, how_many, offset, model_settings, background_frequency,
## background_volume_range, time_shift, mode, sess)
########################################################################
train_fingerprints, train_ground_truth = audio_processor.get_data_my(
FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency,
FLAGS.background_volume, time_shift_samples, 'training', sess)
#mid = np.abs(np.max(train_fingerprints) + np.min(train_fingerprints)) / 2
#half = np.max(train_fingerprints) - np.min(train_fingerprints)
train_fingerprints = np.round(train_fingerprints)
#### 输入归一化 ####
# train_fingerprints=input_normalization(train_fingerprints)
# Run the graph with this batch of training data.
train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run(
[
merged_summaries, evaluation_step, cross_entropy_mean,
train_step, increment_global_step
],
feed_dict={
fingerprint_input: train_fingerprints,
ground_truth_input: train_ground_truth,
learning_rate_input: learning_rate_value,
istrain: True
})
train_writer.add_summary(train_summary, training_step)
tf.logging.info(
'Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' %
(training_step, learning_rate_value, train_accuracy * 100,
cross_entropy_value))
is_last_step = (training_step == training_steps_max)
if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step:
set_size = audio_processor.set_size('validation')
total_accuracy = 0
total_conf_matrix = None
#############################################
########交叉验证集重复计算正确率和混淆矩阵######
for i in xrange(0, set_size, FLAGS.batch_size):
validation_fingerprints, validation_ground_truth = (
audio_processor.get_data_my(FLAGS.batch_size, i,
model_settings, 0.0, 0.0, 0,
'validation', sess))
#mid = np.abs(np.max(validation_fingerprints) + np.min(validation_fingerprints)) / 2
#half = np.max(validation_fingerprints) - np.min(validation_fingerprints)
validation_fingerprints = np.round(validation_fingerprints)
# #### 输入归一化 ####
# validation_fingerprints = input_normalization(validation_fingerprints)
# Run a validation step and capture training summaries for TensorBoard
# with the `merged` op.
validation_summaries, validation_accuracy, conf_matrix = sess.run(
[
validation_merged_summaries, evaluation_step,
confusion_matrix
],
feed_dict={
fingerprint_input: validation_fingerprints,
ground_truth_input: validation_ground_truth,
istrain: True
})
validation_writer.add_summary(validation_summaries,
training_step)
batch_size = min(FLAGS.batch_size, set_size - i)
total_accuracy += (validation_accuracy * batch_size) / set_size
if total_conf_matrix is None:
total_conf_matrix = conf_matrix
else:
total_conf_matrix += conf_matrix
tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' %
(training_step, total_accuracy * 100, set_size))
#############################################
######## 测试集重复计算正确率和混淆矩阵 ######
set_size = audio_processor.set_size('testing')
tf.logging.info('set_size=%d', set_size)
test_fingerprints, test_ground_truth = audio_processor.get_data_my(
-1, 0, model_settings, 0.0, 0.0, 0, 'testing', sess)
#mid = np.abs(np.max(test_fingerprints) + np.min(test_fingerprints)) / 2
#half = np.max(test_fingerprints) - np.min(test_fingerprints)
test_fingerprints = np.round(test_fingerprints)
final_summary, test_accuracy, conf_matrix = sess.run(
[test_summaries, evaluation_step, confusion_matrix],
feed_dict={
fingerprint_input: test_fingerprints,
ground_truth_input: test_ground_truth,
istrain: False
})
final_summary_istrain, test_accuracy_istrain = sess.run(
[test_summaries_istrain, evaluation_step],
feed_dict={
fingerprint_input: test_fingerprints,
ground_truth_input: test_ground_truth,
istrain: True
})
if test_accuracy > best_acc:
best_acc = test_accuracy
best_step = training_step
if test_accuracy_istrain > best_acc_istrain:
best_acc_istrain = test_accuracy_istrain
best_step_istrain = training_step
test_writer.add_summary(final_summary, training_step)
test_istrain_writer.add_summary(final_summary_istrain,
training_step)
tf.logging.info('Confusion Matrix:\n %s' % (conf_matrix))
tf.logging.info('test accuracy = %.1f%% (N=%d)' %
(test_accuracy * 100, 6882))
tf.logging.info('test_istrain accuracy = %.1f%% (N=%d)' %
(test_accuracy_istrain * 100, 6882))
tf.logging.info('Best test accuracy before now = %.1f%% (N=%d)' %
(best_acc * 100, 6882) + ' at step of ' +
str(best_step))
tf.logging.info(
'Best test_istrain accuracy before now = %.1f%% (N=%d)' %
(best_acc_istrain * 100, 6882) + ' at step of ' +
str(best_step_istrain))
# Save the model checkpoint periodically.
if (training_step % FLAGS.save_step_interval == 0
or training_step == training_steps_max):
checkpoint_path = os.path.join(
FLAGS.train_dir + '/' + FLAGS.model_architecture,
FLAGS.model_architecture + '.ckpt')
tf.logging.info('Saving to "%s-%d"', checkpoint_path,
training_step)
saver.save(sess, checkpoint_path, global_step=training_step)
def loss(self):
with tf.variable_scope("loss"):
annotation = tf.expand_dims(self.label,-1,name="annotation")
self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,
labels=self.label)
self.loss = tf.reduce_mean(self.loss)+tf.add_n(tf.get_collection('losses'))
def add_loss(self, pos, neg):
with tf.name_scope('Loss_op'):
loss = tf.reduce_sum(tf.maximum(pos - neg + self.p.margin, 0))
if self.regularizer != None: loss += tf.contrib.layers.apply_regularization(self.regularizer, tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
return loss
def get_trainable_variables(self):
return tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, self.scope)
def train(create_tensor_dict_fn,
create_model_fn,
train_config,
master,
task,
num_clones,
worker_replicas,
clone_on_cpu,
ps_tasks,
worker_job_name,
is_chief,
train_dir,
graph_hook_fn=None):
"""Training function for detection models.
Args:
create_tensor_dict_fn: a function to create a tensor input dictionary.
create_model_fn: a function that creates a DetectionModel and generates
losses.
train_config: a train_pb2.TrainConfig protobuf.
master: BNS name of the TensorFlow master to use.
task: The task id of this training instance.
num_clones: The number of clones to run per machine.
worker_replicas: The number of work replicas to train with.
clone_on_cpu: True if clones should be forced to run on CPU.
ps_tasks: Number of parameter server tasks.
worker_job_name: Name of the worker job.
is_chief: Whether this replica is the chief replica.
train_dir: Directory to write checkpoints and training summaries to.
graph_hook_fn: Optional function that is called after the inference graph is
built (before optimization). This is helpful to perform additional changes
to the training graph such as adding FakeQuant ops. The function should
modify the default graph.
Raises:
ValueError: If both num_clones > 1 and train_config.sync_replicas is true.
"""
detection_model = create_model_fn()
data_augmentation_options = [
preprocessor_builder.build(step)
for step in train_config.data_augmentation_options
]
with tf.Graph().as_default():
# Build a configuration specifying multi-GPU and multi-replicas.
deploy_config = model_deploy.DeploymentConfig(
num_clones=num_clones,
clone_on_cpu=clone_on_cpu,
replica_id=task,
num_replicas=worker_replicas,
num_ps_tasks=ps_tasks,
worker_job_name=worker_job_name)
# Place the global step on the device storing the variables.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
if num_clones != 1 and train_config.sync_replicas:
raise ValueError('In Synchronous SGD mode num_clones must ',
'be 1. Found num_clones: {}'.format(num_clones))
batch_size = train_config.batch_size // num_clones
if train_config.sync_replicas:
batch_size //= train_config.replicas_to_aggregate
with tf.device(deploy_config.inputs_device()):
input_queue = create_input_queue(
batch_size, create_tensor_dict_fn,
train_config.batch_queue_capacity,
train_config.num_batch_queue_threads,
train_config.prefetch_queue_capacity,
data_augmentation_options)
# Gather initial summaries.
# TODO(rathodv): See if summaries can be added/extracted from global tf
# collections so that they don't have to be passed around.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
global_summaries = set([])
model_fn = functools.partial(_create_losses,
create_model_fn=create_model_fn,
train_config=train_config)
clones = model_deploy.create_clones(deploy_config, model_fn,
[input_queue])
first_clone_scope = clones[0].scope
if graph_hook_fn:
with tf.device(deploy_config.variables_device()):
graph_hook_fn()
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by model_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
with tf.device(deploy_config.optimizer_device()):
training_optimizer, optimizer_summary_vars = optimizer_builder.build(
train_config.optimizer)
for var in optimizer_summary_vars:
tf.summary.scalar(var.op.name, var, family='LearningRate')
sync_optimizer = None
if train_config.sync_replicas:
training_optimizer = tf.train.SyncReplicasOptimizer(
training_optimizer,
replicas_to_aggregate=train_config.replicas_to_aggregate,
total_num_replicas=worker_replicas)
sync_optimizer = training_optimizer
with tf.device(deploy_config.optimizer_device()):
regularization_losses = (
None if train_config.add_regularization_loss else [])
total_loss, grads_and_vars = model_deploy.optimize_clones(
clones,
training_optimizer,
regularization_losses=regularization_losses)
total_loss = tf.check_numerics(total_loss,
'LossTensor is inf or nan.')
# Optionally multiply bias gradients by train_config.bias_grad_multiplier.
if train_config.bias_grad_multiplier:
biases_regex_list = ['.*/biases']
grads_and_vars = variables_helper.multiply_gradients_matching_regex(
grads_and_vars,
biases_regex_list,
multiplier=train_config.bias_grad_multiplier)
# Optionally freeze some layers by setting their gradients to be zero.
if train_config.freeze_variables:
grads_and_vars = variables_helper.freeze_gradients_matching_regex(
grads_and_vars, train_config.freeze_variables)
# Optionally clip gradients
if train_config.gradient_clipping_by_norm > 0:
with tf.name_scope('clip_grads'):
grads_and_vars = slim.learning.clip_gradient_norms(
grads_and_vars, train_config.gradient_clipping_by_norm)
# Create gradient updates.
grad_updates = training_optimizer.apply_gradients(
grads_and_vars, global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops, name='update_barrier')
with tf.control_dependencies([update_op]):
train_tensor = tf.identity(total_loss, name='train_op')
# Add summaries.
for model_var in slim.get_model_variables():
global_summaries.add(
tf.summary.histogram('ModelVars/' + model_var.op.name,
model_var))
for loss_tensor in tf.losses.get_losses():
global_summaries.add(
tf.summary.scalar('Losses/' + loss_tensor.op.name,
loss_tensor))
global_summaries.add(
tf.summary.scalar('Losses/TotalLoss', tf.losses.get_total_loss()))
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
summaries |= global_summaries
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# Soft placement allows placing on CPU ops without GPU implementation.
session_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.6))
# Save checkpoints regularly.
keep_checkpoint_every_n_hours = train_config.keep_checkpoint_every_n_hours
saver = tf.train.Saver(
keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
max_to_keep=2)
# Create ops required to initialize the model from a given checkpoint.
init_fn = None
if train_config.fine_tune_checkpoint:
if not train_config.fine_tune_checkpoint_type:
# train_config.from_detection_checkpoint field is deprecated. For
# backward compatibility, fine_tune_checkpoint_type is set based on
# from_detection_checkpoint.
if train_config.from_detection_checkpoint:
train_config.fine_tune_checkpoint_type = 'detection'
else:
train_config.fine_tune_checkpoint_type = 'classification'
var_map = detection_model.restore_map(
fine_tune_checkpoint_type=train_config.
fine_tune_checkpoint_type,
load_all_detection_checkpoint_vars=(
train_config.load_all_detection_checkpoint_vars))
available_var_map = (
variables_helper.get_variables_available_in_checkpoint(
var_map,
train_config.fine_tune_checkpoint,
include_global_step=False))
init_saver = tf.train.Saver(available_var_map)
def initializer_fn(sess):
init_saver.restore(sess, train_config.fine_tune_checkpoint)
init_fn = initializer_fn
slim.learning.train(
train_tensor,
logdir=train_dir,
master=master,
is_chief=is_chief,
session_config=session_config,
startup_delay_steps=train_config.startup_delay_steps,
init_fn=init_fn,
summary_op=summary_op,
number_of_steps=(train_config.num_steps
if train_config.num_steps else None),
save_summaries_secs=120,
# save_interval_secs=300,
sync_optimizer=sync_optimizer,
saver=saver)
def calculate_loss(predict, labels):
landmark_label = labels[:, 0:136]
pose_label = labels[:, 136:139]
leye_cla_label = labels[:, 139]
reye_cla_label = labels[:, 140]
mouth_cla_label = labels[:, 141]
big_mouth_cla_label = labels[:, 142]
landmark_predict = predict[:, 0:136]
pose_predict = predict[:, 136:139]
leye_cla_predict = predict[:, 139]
reye_cla_predict = predict[:, 140]
mouth_cla_predict = predict[:, 141]
big_mouth_cla_predict = predict[:, 142]
loss = _wing_loss(landmark_predict, landmark_label)
loss_pose = _mse(pose_predict, pose_label)
loss=loss_pose+loss
leye_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=leye_cla_predict,
labels=leye_cla_label))
reye_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=reye_cla_predict,
labels=reye_cla_label))
mouth_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=mouth_cla_predict,
labels=mouth_cla_label))
mouth_loss_big = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=big_mouth_cla_predict,
labels=big_mouth_cla_label))
mouth_loss=mouth_loss+mouth_loss_big
###make crosssentropy
leye_cla_correct_prediction = tf.equal(
tf.cast(tf.greater_equal(tf.nn.sigmoid(leye_cla_predict), 0.5), tf.int32),
tf.cast(leye_cla_label, tf.int32))
leye_cla_accuracy = tf.reduce_mean(tf.cast(leye_cla_correct_prediction, tf.float32))
reye_cla_correct_prediction = tf.equal(
tf.cast(tf.greater_equal(tf.nn.sigmoid(reye_cla_predict), 0.5), tf.int32),
tf.cast(reye_cla_label, tf.int32))
reye_cla_accuracy = tf.reduce_mean(tf.cast(reye_cla_correct_prediction, tf.float32))
mouth_cla_correct_prediction = tf.equal(
tf.cast(tf.greater_equal(tf.nn.sigmoid(mouth_cla_predict), 0.5), tf.int32),
tf.cast(mouth_cla_label, tf.int32))
mouth_cla_accuracy = tf.reduce_mean(tf.cast(mouth_cla_correct_prediction, tf.float32))
regularization_losses = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES), name='l2_loss')
if cfg.MODEL.pruning:
bn_l1_loss = []
bn_reg = slim.l1_regularizer(cfg.MODEL.pruning_bn_reg)
variables_restore = tf.get_collection(tf.GraphKeys.MODEL_VARIABLES, scope=cfg.MODEL.net_structure)
for var in variables_restore:
if 'beta' in var.name:
bn_l1_loss.append(bn_reg(var))
l1_loss = tf.add_n(bn_l1_loss, name='l1_loss')
regularization_losses = regularization_losses + l1_loss
return loss, leye_loss, reye_loss, mouth_loss, leye_cla_accuracy, reye_cla_accuracy, mouth_cla_accuracy, regularization_losses
def loss_focal_loss(self):
labels_one_hot = tf.one_hot(self.label,16)
with tf.variable_scope("loss"):
self.loss = focal_loss(prediction_tensor=self.logits,target_tensor=labels_one_hot)
self.loss = tf.reduce_mean(self.loss)+tf.add_n(tf.get_collection('losses'))
def autoencoder(encoder_inputs, decoder_inputs, targets, weights):
emb_encoder_inputs = [
tf.nn.embedding_lookup(self.embedding, i)
for i in encoder_inputs
]
decoder_inputs = [
unk_dropout(i, self.keep_prob, self.vocab.unk_index)
for i in decoder_inputs
]
emb_decoder_inputs = [
tf.nn.embedding_lookup(self.embedding, i)
for i in decoder_inputs
]
assert len(emb_encoder_inputs) == len(emb_decoder_inputs) - 1
assert len(targets) == len(weights)
l2_reg = tf.contrib.layers.l2_regularizer(self.reg_scale)
with tf.variable_scope('autoencoder', regularizer=l2_reg):
_, state = tf.nn.rnn(lstm_cell,
emb_encoder_inputs,
dtype=tf.float32,
scope='rnn_encoder')
with tf.variable_scope('latent'):
# TODO: tf.split(1, 2, linear(state, 2 * latent_dim))
mean = linear(state,
latent_dim,
True,
bias_start=0.0,
scope='mean')
var = tf.nn.softplus(
linear(state,
latent_dim,
True,
bias_start=-3.0,
scope='var')) + 1e-8
batch_size = tf.shape(state[0])[0]
epsilon = tf.random_normal([batch_size, latent_dim])
z = mean + tf.sqrt(var) * epsilon
concat = linear(z, 2 * num_units, True, scope='state')
state = tf.nn.rnn_cell.LSTMStateTuple(*tf.split(1, 2, concat))
proj_w = tf.get_variable('proj_w', [num_units, vocab_size])
proj_b = tf.get_variable('proj_b', [vocab_size])
if forward_only:
loop_function = _extract_argmax_and_embed(
self.embedding, (proj_w, proj_b),
update_embedding=False)
else:
loop_function = None
outputs, _ = tf.nn.seq2seq.rnn_decoder(
emb_decoder_inputs,
state,
lstm_cell,
loop_function=loop_function,
scope='rnn_decoder')
assert same_shape(outputs[0], (None, num_units))
logits = [
tf.nn.xw_plus_b(output, proj_w, proj_b)
for output in outputs
]
assert same_shape(logits[0], (None, vocab_size))
# cross entropy loss = -sum(y * log(p(y))
reconstruction_loss = tf.nn.seq2seq.sequence_loss(
logits, targets, weights)
kl_loss = -0.5 * (1.0 + tf.log(var) - tf.square(mean) - var)
# KL loss averaged by sequence length and batch size
kl_loss = tf.reduce_sum(kl_loss, 1) / (tf.add_n(weights) + 1e-12)
kl_loss = tf.reduce_sum(kl_loss) / tf.cast(batch_size, tf.float32)
annealing_weight = annealing_schedule(
tf.cast(self.global_step, tf.float32), annealing_pivot)
# loss = -E[log(p(x))] + D[q(z)||p(z)]
loss = reconstruction_loss + annealing_weight * kl_loss
if reg_scale > 0.0:
regularizers = tf.add_n(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
loss += regularizers
return logits, loss, (reconstruction_loss, kl_loss,
annealing_weight)
def get_variables(self):
return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.scope)