def define_training(model, args):
# define optimizer
input_lr = tf.placeholder(
tf.float32, shape=[]) # placeholder for dynamic learning rate
model.a('input_lr', input_lr)
if args.opt == 'sgd':
optimizer = tf.train.MomentumOptimizer(
input_lr, 0
) # momentum should not make a difference in calculating gradients
elif args.opt == 'rmsprop':
optimizer = tf.train.RMSPropOptimizer(input_lr)
elif args.opt == 'adam':
optimizer = tf.train.AdamOptimizer(input_lr)
model.a('optimizer', optimizer)
# This adds prob, cross_ent, loss_cross_ent, class_prediction,
# prediction_correct, accuracy, loss, (loss_reg) in tf_nets/losses.py
add_classification_losses(model, model.input_labels)
model.a('train_step', optimizer.minimize(model.loss))
# gradients
grads_and_vars = optimizer.compute_gradients(
model.loss_cross_ent, model.trainable_weights,
model.optimizer.GATE_GRAPH) # change if you want to include l2 loss
model.a('grads_to_compute', [grad for grad, _ in grads_and_vars])
def define_training(model, args):
# define optimizer
input_lr = tf.placeholder(tf.float32, shape=[]) # placeholder for dynamic learning rate
model.a('input_lr', input_lr)
if args.opt == 'sgd':
optimizer = tf.train.MomentumOptimizer(input_lr, args.mom)
elif args.opt == 'rmsprop':
optimizer = tf.train.RMSPropOptimizer(input_lr, momentum=args.mom)
elif args.opt == 'adam':
optimizer = tf.train.AdamOptimizer(input_lr)
model.a('optimizer', optimizer)
# This adds prob, cross_ent, loss_cross_ent, class_prediction,
# prediction_correct, accuracy, loss, (loss_reg) in tf_nets/losses.py
add_classification_losses(model, model.input_labels)
model.a('train_step', optimizer.minimize(model.loss, var_list=model.trainable_weights))
if len(args.freeze_layers) > 0: # freeze certain layers
inds_to_ignore = [int(i) for i in args.freeze_layers.split(',')]
inds_to_train = np.delete(np.arange(len(model.trainable_weights)), inds_to_ignore)
print('Only training layers:', inds_to_train)
model.a('train_step_freeze', optimizer.minimize(model.loss, var_list=[model.trainable_weights[i] for i in inds_to_train]))
if len(args.opt2_layers) > 0: # use 2 different optimizers for different layers
# This only uses a constant LR, no decaying LR
if args.opt == 'sgd':
optimizer_special = tf.train.MomentumOptimizer(args.lr_special, args.mom_special)
elif args.opt == 'rmsprop':
optimizer_special = tf.train.RMSPropOptimizer(args.lr_special, momentum=args.mom_special)
elif args.opt == 'adam':
optimizer_special = tf.train.AdamOptimizer(args.lr_special)
model.a('optimizer_special', optimizer_special)
inds_for_opt2 = [int(i) for i in args.opt2_layers.split(',')]
inds_for_opt1 = np.delete(np.arange(len(model.trainable_weights)), inds_for_opt2)
print('These layers get opt1:', inds_for_opt1)
print('These layers get opt2:', inds_for_opt2)
model.a('train_step_1', optimizer.minimize(model.loss, var_list=[model.trainable_weights[i] for i in inds_for_opt1]))
model.a('train_step_2', optimizer_special.minimize(model.loss, var_list=[model.trainable_weights[i] for i in inds_for_opt2]))
# gradients (may be used for mini-batches)
grads_and_vars = optimizer.compute_gradients(model.loss, model.trainable_weights)
model.a('grads_to_compute', [grad for grad, _ in grads_and_vars])
print('All model weights:')
summarize_weights(model.trainable_weights)
print('trainable:')
for x in model.trainable_weights:
print(x)
print('non trainable:')
for x in model.non_trainable_weights:
print(x)
print('saved to weights file:')
for x in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
print(x)
def define_training(model, args):
# define optimizer
input_lr = tf.placeholder(
tf.float32, shape=[]) # placeholder for dynamic learning rate
model.a('input_lr', input_lr)
if args.opt == 'sgd':
optimizer = tf.train.MomentumOptimizer(input_lr, args.mom)
elif args.opt == 'rmsprop':
optimizer = tf.train.RMSPropOptimizer(input_lr, momentum=args.mom)
elif args.opt == 'adam':
optimizer = tf.train.AdamOptimizer(input_lr)
model.a('optimizer', optimizer)
# This adds prob, cross_ent, loss_cross_ent, class_prediction,
# prediction_correct, accuracy, loss, (loss_reg) in tf_nets/losses.py
add_classification_losses(model, model.input_labels)
model.a('train_step',
optimizer.minimize(model.loss, var_list=model.trainable_weights))
# gradients (may be used for mini-batches)
grads_and_vars = optimizer.compute_gradients(model.loss,
model.trainable_weights)
model.a('grads_to_compute', [grad for grad, _ in grads_and_vars])
print('All model weights:')
summarize_weights(model.trainable_weights)
print('trainable:')
for x in model.trainable_weights:
print(x)
print('non trainable:')
for x in model.non_trainable_weights:
print(x)
print('saved to weights file:')
for x in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
print(x)
print('grad summaries:')
add_grad_summaries(grads_and_vars)
print('opt summary:')
summarize_opt(optimizer)