def create_cost(c, w, b, nb, length, weight_spacing = 1.0, weight_bending = 1.0, gamma = 1.0, kappa = 2.0):
#tangents
t = create_tangent(c);
tn = create_normalize_tangent(t);
nl = create_normal(tn);
nr = tf.scalar_mul(-1.0, nl);
l,r = create_left_right(c,w,nl);
cost_left = create_cost_soft_min_aligned_distance(l, b, nl, nb, k = kappa, gamma = gamma);
cost_right= create_cost_soft_min_aligned_distance(r, b, nr, nb, k = kappa, gamma = gamma);
cost = tf.add(cost_left, cost_right);
#spacing and bending
if weight_spacing != 0:
cost_spacing = tf.scalar_mul(weight_spacing, create_cost_spacing(t, length));
cost = tf.add(cost, cost_spacing);
else:
cost_spacing = tf.constant(0);
if weight_bending != 0:
cost_bending = tf.scalar_mul(weight_bending, create_cost_bending(tn));
cost = tf.add(cost, cost_bending);
else:
cost_bending = tf.constant(0);
return (cost, cost_left, cost_right, cost_spacing, cost_bending, nl, l, r);
def clip_norm(g, c, n):
if c <= 0: # if clipnorm == 0 no need to add ops to the graph
return g
# tf require using a special op to multiply IndexedSliced by scalar
if K.backend() == 'tensorflow':
condition = n >= c
then_expression = tf.scalar_mul(c / n, g)
else_expression = g
# saving the shape to avoid converting sparse tensor to dense
if isinstance(then_expression, tf.Tensor):
g_shape = copy.copy(then_expression.get_shape())
elif isinstance(then_expression, tf.IndexedSlices):
g_shape = copy.copy(then_expression.dense_shape)
if condition.dtype != tf.bool:
condition = tf.cast(condition, 'bool')
g = tf.cond(condition,
lambda: then_expression,
lambda: else_expression)
if isinstance(then_expression, tf.Tensor):
g.set_shape(g_shape)
elif isinstance(then_expression, tf.IndexedSlices):
g._dense_shape = g_shape
else:
g = K.switch(K.greater_equal(n, c), g * c / n, g)
return g
def predict_slim(sample_images, print_func=print):
"""
Code modified from here: [https://github.com/tensorflow/models/issues/429]
"""
# Setup preprocessing
input_tensor = tf.placeholder(tf.float32, shape=(None, 299, 299, 3), name='input_image')
scaled_input_tensor = tf.scalar_mul((1.0 / 255), input_tensor)
scaled_input_tensor = tf.subtract(scaled_input_tensor, 0.5)
scaled_input_tensor = tf.multiply(scaled_input_tensor, 2.0)
# Setup session
sess = tf.Session()
arg_scope = slim_irv2.inception_resnet_v2_arg_scope()
with slim.arg_scope(arg_scope):
_, end_points = slim_irv2.inception_resnet_v2(scaled_input_tensor, is_training=False)
# Load the model
print_func("Loading TF-slim checkpoint...")
saver = tf.train.Saver()
saver.restore(sess, SLIM_CKPT)
# Make prediction
predict_values = []
for image in sample_images:
im = Image.open(image).resize((299, 299))
arr = np.expand_dims(np.array(im), axis=0)
y_pred = sess.run([end_points['Predictions']], feed_dict={input_tensor: arr})
y_pred = y_pred[0].ravel()
y_pred = y_pred[1:] / y_pred[1:].sum() # remove background class and renormalize
print_func("{} class={} prob={}".format(image, np.argmax(y_pred), np.max(y_pred)))
predict_values.append(y_pred)
return predict_values
def thresholding(inputs):
# find the mean for each example in the batch
mean_output = tf.reduce_mean(inputs, axis=1)
# scale each mean based on a factor
threshold_scalar = tf.Variable(utils.threshold_scalar, tf.float32)
scaled_mean = tf.scalar_mul(threshold_scalar, mean_output)
scaled_mean = tf.reshape(scaled_mean, [utils.batch_size])
# setup matrix for
min_thresh_for_max = tf.fill([utils.batch_size], 0.05)
max_thresh_for_min = tf.fill([utils.batch_size], 0.15) #0.4
thresholds = tf.maximum(min_thresh_for_max, scaled_mean)
thresholds = tf.minimum(max_thresh_for_min, thresholds)
# zero values under the thresholds using bitmask
thresholds = tf.reshape(thresholds, [128, 1, 1])
threshold_mask = tf.cast(tf.greater(inputs, thresholds), tf.float32)
thresholded_input = tf.multiply(inputs, threshold_mask)
# peak picking
# select beats by x[i-1] < x[i] > x[i+1] (local maximum)
x_minus_1 = tf.cast(tf.greater(thresholded_input, tf.manip.roll(thresholded_input, shift=-1, axis=1)), tf.float32)
x_plus_1 = tf.cast(tf.greater(thresholded_input, tf.manip.roll(thresholded_input, shift=1, axis=1)), tf.float32)
output = tf.multiply(x_minus_1, x_plus_1)
return output
def focal_loss(onehot_labels, cls_preds,
alpha=0.25, gamma=2.0, name=None, scope=None):
"""Compute softmax focal loss between logits and onehot labels
logits and onehot_labels must have same shape [batchsize, num_classes] and
the same data type (float16, 32, 64)
Args:
onehot_labels: Each row labels[i] must be a valid probability distribution
cls_preds: Unscaled log probabilities
alpha: The hyperparameter for adjusting biased samples, default is 0.25
gamma: The hyperparameter for penalizing the easy labeled samples
name: A name for the operation (optional)
Returns:
A 1-D tensor of length batch_size of same type as logits with softmax focal loss
"""
with tf.name_scope(scope, 'focal_loss', [cls_preds, onehot_labels]) as sc:
logits = tf.convert_to_tensor(cls_preds)
onehot_labels = tf.convert_to_tensor(onehot_labels)
precise_logits = tf.cast(logits, tf.float32) if (
logits.dtype == tf.float16) else logits
onehot_labels = tf.cast(onehot_labels, precise_logits.dtype)
predictions = tf.nn.sigmoid(logits)
predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1. - predictions)
# add small value to avoid 0
epsilon = 1e-8
alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1 - alpha_t)
losses = tf.reduce_sum(
-alpha_t * tf.pow(1. - predictions_pt, gamma) * onehot_labels * tf.log(predictions_pt + epsilon),
name=name, axis=1)
return losses
def build_graph(self,test_decoder_logits):
print('starting building graph [sentiment-discriminator]')
with tf.variable_scope("sentiment") as scope:
self.inputs = tf.slice(test_decoder_logits,[0,0,0],[self.batch_size,self.max_length,self.vocab_size])
# variable
weights = {
'w2v' : tf.get_variable(initializer = tf.random_uniform_initializer(-0.1, 0.1, dtype=tf.float32),shape = [self.vocab_size, self.embedding_dim], name='w2v'),
'out_1' : tf.get_variable(initializer = tf.random_normal_initializer(), shape = [self.unit_size*2, 1], name='w_out_1'),
}
biases = {
'out_1' : tf.get_variable(initializer = tf.random_normal_initializer(), shape=[1], name='b_out_1'),
}
# structure
def BiRNN(x):
x = tf.unstack(x, self.max_length, 1)
lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(self.unit_size, forget_bias=1.0)
lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(self.unit_size,forget_bias=1.0)
outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32 )
return outputs[-1]
self.inputs_softmax = tf.nn.softmax(tf.scalar_mul(tf.constant(5.0, shape=[]),self.inputs))
y_list=[]
for i in range(self.inputs.get_shape().as_list()[0]):
y = tf.matmul(self.inputs_softmax[i], weights['w2v'])
y = tf.reshape(y, [1, self.max_length, self.embedding_dim])
y_list.append(y)
embbed_layer = tf.concat(y_list,0)
layer_1 = BiRNN(embbed_layer)
pred = tf.matmul(layer_1, weights['out_1']) + biases['out_1']
# get score
self.score = tf.sigmoid(pred)
def test_decoder_loop(prev,i):
factor = tf.constant(5,shape=(),dtype=tf.float32)
prev = tf.scalar_mul(factor,tf.add(tf.matmul(prev,weight_output),bias_output))
prev_index = tf.nn.softmax(prev)
pred_prev = tf.matmul(prev_index,word_embedding_matrix)
next_input = pred_prev
return next_input
def init_training_graph(self):
with tf.name_scope('Evaluation'):
logits = self.last
prob_b = tf.squeeze(logits, squeeze_dims=[1,2])
self.predictions = tf.argmax(prob_b, axis=1)
with tf.name_scope('Loss'):
self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prob_b,
labels=tf.cast(self.train_labels_node, tf.int32),
name="entropy")))
tf.summary.scalar("entropy", self.loss)
with tf.name_scope('Accuracy'):
LabelInt = tf.cast(self.train_labels_node, tf.int64)
CorrectPrediction = tf.equal(self.predictions, LabelInt)
self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32))
tf.summary.scalar("accuracy", self.accuracy)
with tf.name_scope('Prediction'):
self.TP = tf.count_nonzero(self.predictions * LabelInt)
self.TN = tf.count_nonzero((self.predictions - 1) * (LabelInt - 1))
self.FP = tf.count_nonzero(self.predictions * (LabelInt - 1))
self.FN = tf.count_nonzero((self.predictions - 1) * LabelInt)
with tf.name_scope('Precision'):
self.precision = tf.divide(self.TP, tf.add(self.TP, self.FP))
tf.summary.scalar('Precision', self.precision)
with tf.name_scope('Recall'):
self.recall = tf.divide(self.TP, tf.add(self.TP, self.FN))
tf.summary.scalar('Recall', self.recall)
with tf.name_scope('F1'):
num = tf.multiply(self.precision, self.recall)
dem = tf.add(self.precision, self.recall)
self.F1 = tf.scalar_mul(2, tf.divide(num, dem))
tf.summary.scalar('F1', self.F1)
with tf.name_scope('MeanAccuracy'):
Nprecision = tf.divide(self.TN, tf.add(self.TN, self.FN))
self.MeanAcc = tf.divide(tf.add(self.precision, Nprecision) ,2)
#self.batch = tf.Variable(0, name = "batch_iterator")
self.train_prediction = tf.nn.softmax(logits)
self.test_prediction = tf.nn.softmax(logits)
tf.global_variables_initializer().run()
print('Computational graph initialised')
def create_left_right(c, w, nrm): left = tf.transpose(tf.mul(tf.transpose(nrm), w)); right= tf.scalar_mul(-1.0, left); left = tf.add(left, c); right= tf.add(right,c); return left,right
def main(_):
print('loading word embeddings from %s' % FLAGS.embedding_file)
weight_matrix, word_idx = sentiment.load_embeddings(FLAGS.embedding_file)
train_file = os.path.join(FLAGS.tree_dir, 'train.txt')
print('loading training trees from %s' % train_file)
train_trees = sentiment.load_trees(train_file)
dev_file = os.path.join(FLAGS.tree_dir, 'dev.txt')
print('loading dev trees from %s' % dev_file)
dev_trees = sentiment.load_trees(dev_file)
with tf.Session() as sess:
print('creating the model')
keep_prob = tf.placeholder_with_default(1.0, [])
train_feed_dict = {keep_prob: FLAGS.keep_prob}
word_embedding = sentiment.create_embedding(weight_matrix)
compiler, metrics = sentiment.create_model(
word_embedding, word_idx, FLAGS.lstm_num_units, keep_prob)
loss = tf.reduce_sum(compiler.metric_tensors['all_loss'])
opt = tf.train.AdagradOptimizer(FLAGS.learning_rate)
grads_and_vars = opt.compute_gradients(loss)
found = 0
for i, (grad, var) in enumerate(grads_and_vars):
if var == word_embedding.weights:
found += 1
grad = tf.scalar_mul(FLAGS.embedding_learning_rate_factor, grad)
grads_and_vars[i] = (grad, var)
assert found == 1 # internal consistency check
train = opt.apply_gradients(grads_and_vars)
saver = tf.train.Saver()
print('initializing tensorflow')
sess.run(tf.global_variables_initializer())
with compiler.multiprocessing_pool():
print('training the model')
train_set = compiler.build_loom_inputs(train_trees)
dev_feed_dict = compiler.build_feed_dict(dev_trees)
dev_hits_best = 0.0
for epoch, shuffled in enumerate(td.epochs(train_set, FLAGS.epochs), 1):
train_loss = 0.0
for batch in td.group_by_batches(shuffled, FLAGS.batch_size):
train_feed_dict[compiler.loom_input_tensor] = batch
_, batch_loss = sess.run([train, loss], train_feed_dict)
train_loss += batch_loss
dev_metrics = sess.run(metrics, dev_feed_dict)
dev_loss = dev_metrics['all_loss']
dev_accuracy = ['%s: %.2f' % (k, v * 100) for k, v in
sorted(dev_metrics.items()) if k.endswith('hits')]
print('epoch:%4d, train_loss: %.3e, dev_loss: %.3e, dev_accuracy: [%s]'
% (epoch, train_loss, dev_loss, ' '.join(dev_accuracy)))
dev_hits = dev_metrics['root_hits']
if dev_hits > dev_hits_best:
dev_hits_best = dev_hits
save_path = saver.save(sess, FLAGS.checkpoint_base, global_step=epoch)
print('model saved in file: %s' % save_path)
def _add_focal_losses(self, sigma_rpn=3.0):
with tf.variable_scope('loss_' + self._tag) as scope:
# RPN, class loss
rpn_cls_score = tf.reshape(self._predictions['rpn_cls_score_reshape'], [-1, 2])
rpn_label = tf.reshape(self._anchor_targets['rpn_labels'], [-1])
rpn_select = tf.where(tf.not_equal(rpn_label, -1))
rpn_cls_score = tf.reshape(tf.gather(rpn_cls_score, rpn_select), [-1, 2])
rpn_label = tf.reshape(tf.gather(rpn_label, rpn_select), [-1])
rpn_cross_entropy = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=rpn_cls_score, labels=rpn_label))
# RPN, bbox loss
rpn_bbox_pred = self._predictions['rpn_bbox_pred']
rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets']
rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights']
rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights']
rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights,
rpn_bbox_outside_weights, sigma=sigma_rpn, dim=[1, 2, 3])
# RCNN, class loss
alpha_scale = 0.25
gamma = 2
epsilon = 1e-8
cls_score = self._predictions["cls_score"]
label = tf.reshape(self._proposal_targets["labels"], [-1])
label = tf.one_hot(label, depth=self._num_classes)
cls_pred = tf.nn.sigmoid(cls_score)
predictions_pt = tf.where(tf.equal(label, 1), cls_pred, 1-cls_pred)
alpha_t = tf.ones_like(label, dtype=tf.float32)
alpha_t = tf.scalar_mul(alpha_scale, alpha_t)
alpha_t = tf.where(tf.equal(label, 1.0), alpha_t, 1. - alpha_t)
cross_entropy = tf.reduce_mean(-alpha_t*tf.pow(1 - predictions_pt, gamma)*tf.log(predictions_pt + epsilon))
# RCNN, bbox loss
bbox_pred = self._predictions['bbox_pred']
bbox_targets = self._proposal_targets['bbox_targets']
bbox_inside_weights = self._proposal_targets['bbox_inside_weights']
bbox_outside_weights = self._proposal_targets['bbox_outside_weights']
loss_box = self._smooth_l1_loss(bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights)
self._losses['cross_entropy'] = cross_entropy
self._losses['loss_box'] = loss_box
self._losses['rpn_cross_entropy'] = rpn_cross_entropy
self._losses['rpn_loss_box'] = rpn_loss_box
self._losses['rpn_loss'] = rpn_loss_box + rpn_cross_entropy
self._losses['class_loss'] = cross_entropy + loss_box
loss = cross_entropy + loss_box + rpn_cross_entropy + rpn_loss_box
self._losses['total_loss'] = loss
self._event_summaries.update(self._losses)
return loss
def create_cost(center, xy, width, persitaltic, bend, bend_profiles, contour, contour_normals, length, weight_spacing = 1.0, weight_bending = 1.0, gamma = 1.0, kappa = 2.0):
c = tf.add(center, xy);
#tangent and normals
t = create_tangent(c);
tn = create_normalize_tangent(t);
ta = create_average_tangent(tn);
nl = create_normal(ta);
#bend the center
c = create_bend(center, nl, bend, bend_profiles);
#peristaltically move the center
c = create_peristaltic(c, ta, persitaltic);
#tangents
t = create_tangent(c);
tn = create_normalize_tangent(t);
ta = create_average_tangent(tn);
nl = create_normal(ta);
nr = tf.scalar_mul(-1.0, nl);
l,r = create_left_right(c,width,nl);
cost_left = create_cost_soft_min_aligned_distance(l, contour, nl, contour_normals, k = kappa, gamma = gamma);
cost_right= create_cost_soft_min_aligned_distance(r, contour, nr, contour_normals, k = kappa, gamma = gamma);
cost = tf.add(cost_left, cost_right);
#spacing and bending
if weight_spacing != 0:
cost_spacing = tf.scalar_mul(weight_spacing, create_cost_spacing(t, length));
cost = tf.add(cost, cost_spacing);
else:
cost_spacing = tf.constant(0);
if weight_bending != 0:
cost_bending = tf.scalar_mul(weight_bending, create_cost_bending(tn));
cost = tf.add(cost, cost_bending);
else:
cost_bending = tf.constant(0);
return (cost, cost_left, cost_right, cost_spacing, cost_bending, c, l, r, nl);
def focal_loss3(cls_score, label, num_classes):
alpha_scale = 0.25
gamma = 2
epsilon = 1e-8
label = tf.one_hot(label, depth=num_classes)
cls_pred = tf.nn.sigmoid(cls_score)
predictions_pt = tf.where(tf.equal(label, 1), cls_pred, 1 - cls_pred)
alpha_t = tf.ones_like(label, dtype=tf.float32)
alpha_t = tf.scalar_mul(alpha_scale, alpha_t)
alpha_t = tf.where(tf.equal(label, 1.0), alpha_t, 1. - alpha_t)
losses = tf.reduce_mean(-alpha_t * tf.pow(1 - predictions_pt, gamma) * tf.log(predictions_pt + epsilon), axis=1)
return losses
def build_model(self):
self.images = tf.placeholder(tf.float32, [self.batch_size] + [self.output_size, self.output_size, self.c_dim], name='real_images')
self.sample_images= tf.placeholder(tf.float32, [self.sample_size] + [self.output_size, self.output_size, self.c_dim], name='sample_images')
self.z = tf.placeholder(tf.float32, [None, self.z_dim], name='z')
self.z_sum = tf.summary.histogram("z", self.z)
self.G = self.generator(self.z)
self.D = self.discriminator(self.images)
self.D_ = self.discriminator(self.G, reuse=True)
self.sampler = self.sampler(self.z)
self.d_sum = tf.summary.histogram("d", self.D)
self.d__sum = tf.summary.histogram("d_", self.D_)
self.G_sum = tf.summary.image("G", self.G)
self.d_loss_real = tf.reduce_mean(tf.scalar_mul(-1, self.D))
self.d_loss_fake = tf.reduce_mean(self.D_)
self.g_loss = tf.reduce_mean(tf.scalar_mul(-1, self.D_))
self.d_loss_real_sum = tf.summary.scalar("d_loss_real", self.d_loss_real)
self.d_loss_fake_sum = tf.summary.scalar("d_loss_fake", self.d_loss_fake)
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss)
self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.epoch = tf.Variable(-1, name='epoch', trainable=False)
self.increment_epoch = tf.assign(self.epoch, self.epoch+1)
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.saver = tf.train.Saver(max_to_keep=1000)
def entropy_matrix(graph, P):
"""
:param graph:
:param P:
:return:
"""
with graph.as_default():
shape = P.get_shape().as_list()
one_diagonal = tf.diag(diagonal=[1.0 for _ in range(shape[0])])
P_mod = tf.add(P, one_diagonal)
H = tf.reduce_sum(tf.scalar_mul(-1.0, tf.mul(P, tf.log(P_mod))), 1)
return H
def _compute_update(self, param, grad, state):
"""Compute updates of parameters."""
# get the learning rate at the current index, if the index
# is greater than the number of available learning rates,
# use the last one
index = tf.minimum(state["itr"], self.max_index)
learning_rate = tf.gather(self.learning_rates, index)
# update the parameters: parameter - learning_rate * gradient
updated_param = param - tf.scalar_mul(learning_rate, grad)
return updated_param, {"itr": state["itr"] + 1}
def init_training_graph(self):
with tf.name_scope('Evaluation'):
self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/")
self.predictions = tf.argmax(self.logits, axis=3)
with tf.name_scope('Loss'):
self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,
labels=tf.squeeze(tf.cast(self.train_labels_node, tf.int32), squeeze_dims=[3]),
name="entropy")))
tf.summary.scalar("entropy", self.loss)
with tf.name_scope('Accuracy'):
LabelInt = tf.squeeze(tf.cast(self.train_labels_node, tf.int64), squeeze_dims=[3])
CorrectPrediction = tf.equal(self.predictions, LabelInt)
self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32))
tf.summary.scalar("accuracy", self.accuracy)
with tf.name_scope('ClassPrediction'):
flat_LabelInt = tf.reshape(LabelInt, [-1])
flat_predictions = tf.reshape(self.predictions, [-1])
self.cm = tf.confusion_matrix(flat_LabelInt, flat_predictions, self.NUM_LABELS)
flatten_confusion_matrix = tf.reshape(self.cm, [-1])
total = tf.reduce_sum(self.cm)
for i in range(self.NUM_LABELS):
name = "Label_{}".format(i)
TP, TN, FP, FN = GetCMInfo_TF(self.cm, i, self.NUM_LABELS)
precision = tf.divide(TP, tf.add(TP, FP))
recall = tf.divide(TP, tf.add(TP, FN))
num = tf.multiply(precision, recall)
dem = tf.add(precision, recall)
F1 = tf.scalar_mul(2, tf.divide(num, dem))
Nprecision = tf.divide(TN, tf.add(TN, FN))
MeanAcc = tf.divide(tf.add(precision, Nprecision) ,2)
tf.summary.scalar(name + '_Precision', precision)
tf.summary.scalar(name + '_Recall', recall)
tf.summary.scalar(name + '_F1', F1)
tf.summary.scalar(name + '_Performance', MeanAcc)
confusion_image = tf.reshape( tf.cast( self.cm, tf.float32),
[1, self.NUM_LABELS, self.NUM_LABELS, 1])
tf.summary.image('confusion', confusion_image)
self.train_prediction = tf.nn.softmax(self.logits)
self.test_prediction = self.train_prediction
tf.global_variables_initializer().run()
print('Computational graph initialised')
def SimStep(x,In_Yr,In_Yp,In_k,In_dt):
Yr=tf.constant(In_Yr,tf.float32)# Reactant ratios for each reaction
Yp=tf.constant(In_Yp,tf.float32)# Product ratios for each reaction
k=tf.constant(In_k,tf.float32)# Reaction constants for each reaction
dt=tf.constant(In_dt,tf.float32) # time lapse for each simulation step
s1=tf.pow(x,Yr)
s2=tf.reduce_prod(s1,1)
r=k*s2#Reacion rates
s4=tf.scalar_mul(dt,r)
Yd=Yp-Yr # Change in concentrations attribute to each reaction
dxij=s4*tf.transpose(Yd)# concentration changes each reaction in this step***
dx=tf.reduce_sum(dxij,1) #sum of concentration changes from all reactions in this step***
xp=x+dx#New concentration after steps
return(xp)
def __init__(self, x, weights: Weights, b: float, hp: Hyperparameters) -> None:
self.x = tf.transpose(x)
self.weights = weights
self.b = b
with tf.variable_scope('Learning'):
self.alpha_ma_w = tf.scalar_mul(scalar=hp.alpha_ma, x=self.weights.w)
self.one_minus_alpha_ma_x = tf.scalar_mul(scalar=1. - hp.alpha_ma, x=self.x)
self.new_w = self.alpha_ma_w + self.one_minus_alpha_ma_x
self.learn_weights_op = self.weights.assign(self.new_w)
with tf.variable_scope('Layer'):
with tf.variable_scope('Wx_plus_b'):
z = tf.add(tf.matmul(x, weights.w), self.b)
activation = tf.nn.relu(z, name='Activation')
excitatory, inhibitory = weights.split(activation)
self.excitatory_y = excitatory
self.inhibitory_y = tf.scalar_mul(scalar=-1., x = inhibitory)
self.y = tf.concat([self.excitatory_y, self.inhibitory_y], axis=1)
util.variable_summaries(self.y)
print(self.y, self.y.shape)
def AddGaussianNoise(t, sigma, noise_rate, name=None):
"""Add i.i.d. Gaussian noise (0, sigma^2) to every entry of t.
Args:
t: the input tensor.
sigma: the stddev of the Gaussian noise.
name: optional name.
Returns:
the noisy tensor.
"""
with tf.name_scope(values=[t, sigma], name=name,
default_name="add_gaussian_noise") as name:
noisy_t = t + tf.scalar_mul(noise_rate, tf.random_normal(tf.shape(t), stddev=sigma))
return noisy_t
def probability_matrix(graph, beta, D):
"""
:param D:
:return:
"""
with graph.as_default():
shape = D.get_shape().as_list()
beta_matrix = tf.tile(tf.expand_dims(beta, 1), multiples=[1, shape[1]])
sqrt_matrix = tf.tile(tf.constant(0.5, dtype=tf.float32, shape=[1,1]), multiples=[shape[0], shape[1]])
zero_diagonal = tf.exp(tf.diag(diagonal=[-np.inf for _ in range(shape[0])]))
exp = tf.mul(tf.exp(tf.mul(beta_matrix, tf.scalar_mul(-1.0, tf.pow(D, sqrt_matrix)))), zero_diagonal)
sum_exp = tf.reduce_sum(exp, reduction_indices=1)
sum_exp_matrix = tf.tile(tf.expand_dims(sum_exp, 1), multiples=[1, shape[1]])
return tf.div(exp, sum_exp_matrix)
def calculate_euclidean_distance(training_data, test_data):
# computing (x-y)^2 = x^2 + y^2 - 2xy
first_term = tf.reduce_sum(training_data**2, axis=1)
second_term = tf.expand_dims(tf.reduce_sum(test_data**2, axis=1), axis=1)
xy = tf.matmul(test_data, tf.transpose(training_data))
third_term = tf.scalar_mul(-2, xy)
dist = first_term + second_term + third_term
# shape1 = tf.shape(first_term)
# shape2 = tf.shape(second_term)
# shape3 = tf.shape(third_term)
# sess = tf.InteractiveSession()
# print(sess.run([dist], feed_dict={X: trainData, Z: testData}))
# print(sess.run([shape1, shape2, shape3], feed_dict={X: trainData, Z: testData}))
return dist
def apply_weightshared_on_grads(grads_and_vars, dict_cenidx):
# Mask gradients with pruned elements
import config
for key, cenidx in dict_cenidx.items():
count = 0
for grad, var in grads_and_vars:
if var.name == key+":0":
num_cen = config.kmeans_para[key]
tem_grad=[]
for i in range(num_cen):
cenidx_obj = tf.cast(tf.constant(cenidx == i), tf.float32)
tem_grad.append(tf.scalar_mul(tf.reduce_sum(tf.multiply(cenidx_obj,grad)),cenidx_obj))
grad=(tf.add_n(tem_grad))
grads_and_vars[count] = (grad, var)
count += 1
return grads_and_vars
def get_batch(filename, batch_size):
if not tf.gfile.Exists(filename):
raise ValueError('Failed to find file ' + filename)
# get single examples
label, image = _read_and_decode_single_example(filename)
# groups examples into batches randomly
images_batch, labels_batch = tf.train.shuffle_batch(
[image, label], batch_size=batch_size,
capacity=2000,
min_after_dequeue=1000)
images_batch = tf.cast(images_batch, tf.float32)
norm_factor = tf.constant(1.0/255)
images_batch = tf.scalar_mul(norm_factor, images_batch)
#labels_batch = tf.cast(labels_batch, tf.float32)
return images_batch, labels_batch
def distance_matrix(graph, X):
"""
:param X: Input Matrix for which to calculate Distance Matrix
:return:
"""
with graph.as_default():
shape = X.get_shape().as_list()
Di = tf.reduce_sum(tf.mul(X, X), 1)
I_magnitude_tiled = tf.tile(tf.expand_dims(Di, 1), multiples=[1, shape[0]])
I_t_magnitude_tiled = tf.tile(tf.expand_dims(Di, 0), multiples=[shape[0], 1])
I_I = tf.matmul(X, tf.transpose(X))
D = tf.add(
tf.add(I_magnitude_tiled, I_t_magnitude_tiled),
tf.scalar_mul(-2.0, I_I)
)
zero_diagonal = tf.exp(tf.diag(diagonal=[-np.inf for _ in range(shape[0])]))
return tf.mul(D, zero_diagonal)
def create_cost_soft_min_distance(self, c, s): """Creates a soft-min distance of the centers to the points""" c_shape = c.get_shape().as_list(); s_shape = s.get_shape().as_list(); #expand matrices cc = tf.reshape(c, [c_shape[0], c_shape[1], 1]); ss = tf.reshape(s, [s_shape[0], s_shape[1], 1]); ss = tf.transpose(ss, perm = [0,2,1]); cc = tf.tile(cc, [1, 1, s_shape[0]]); ss = tf.tile(ss, [c_shape[0], 1, 1]); #pairwise distances dist2 = tf.sqrt(tf.reduce_sum(tf.squared_difference(cc,ss), reduction_indices = 1)); dist2 = tf.reduce_mean(dist2, reduction_indices=0); # hack: get rid of batches here #softmin return tf.reduce_sum(tf.mul(tf.nn.softmax(tf.scalar_mul(tf.constant(-1.0,"float32"), dist2)), dist2),reduction_indices = 0);
def clip_norm(g, c, n):
"""Clip the gradient `g` if the L2 norm `n` exceeds `c`.
# Arguments
g: Tensor, the gradient tensor
c: float >= 0. Gradients will be clipped
when their L2 norm exceeds this value.
n: Tensor, actual norm of `g`.
# Returns
Tensor, the gradient clipped if required.
"""
if c <= 0: # if clipnorm == 0 no need to add ops to the graph
return g
# tf require using a special op to multiply IndexedSliced by scalar
if K.backend() == 'tensorflow':
condition = n >= c
then_expression = tf.scalar_mul(c / n, g)
else_expression = g
# saving the shape to avoid converting sparse tensor to dense
if isinstance(then_expression, tf.Tensor):
g_shape = copy.copy(then_expression.get_shape())
elif isinstance(then_expression, tf.IndexedSlices):
g_shape = copy.copy(then_expression.dense_shape)
if condition.dtype != tf.bool:
condition = tf.cast(condition, 'bool')
g = tf.cond(condition,
lambda: then_expression,
lambda: else_expression)
if isinstance(then_expression, tf.Tensor):
g.set_shape(g_shape)
elif isinstance(then_expression, tf.IndexedSlices):
g._dense_shape = g_shape
else:
g = K.switch(K.greater_equal(n, c), g * c / n, g)
return g
def clip_norm(g, c, n):
if c > 0:
if K.backend() == 'tensorflow':
import tensorflow as tf
import copy
condition = n >= c
then_expression = tf.scalar_mul(c / n, g)
else_expression = g
if hasattr(then_expression, 'get_shape'):
g_shape = copy.copy(then_expression.get_shape())
elif hasattr(then_expression, 'dense_shape'):
g_shape = copy.copy(then_expression.dense_shape)
if condition.dtype != tf.bool:
condition = tf.cast(condition, 'bool')
g = K.tensorflow_backend.control_flow_ops.cond(
condition, lambda: then_expression, lambda: else_expression)
if hasattr(then_expression, 'get_shape'):
g.set_shape(g_shape)
elif hasattr(then_expression, 'dense_shape'):
g._dense_shape = g_shape
else:
g = K.switch(n >= c, g * c / n, g)
return g
def train_setup(self):
tf.set_random_seed(self.conf.random_seed)
# Create queue coordinator.
self.coord = tf.train.Coordinator()
# Input size
input_size = (self.conf.input_height, self.conf.input_width)
# Load reader
with tf.name_scope("create_inputs"):
reader = ImageReader(self.conf.data_dir, self.conf.data_list,
input_size, self.conf.random_scale,
self.conf.random_mirror,
self.conf.ignore_label, IMG_MEAN, self.coord)
self.image_batch, self.label_batch = reader.dequeue(
self.conf.batch_size)
# Create network
if self.conf.encoder_name not in ['res101', 'res50', 'deeplab']:
print('encoder_name ERROR!')
print("Please input: res101, res50, or deeplab")
sys.exit(-1)
elif self.conf.encoder_name == 'deeplab':
net = Deeplab_v2(self.image_batch, self.conf.num_classes, True)
# Variables that load from pre-trained model.
restore_var = [
v for v in tf.global_variables() if 'fc' not in v.name
]
# Trainable Variables
all_trainable = tf.trainable_variables()
# Fine-tune part
encoder_trainable = [
v for v in all_trainable if 'fc' not in v.name
] # lr * 1.0
# Decoder part
decoder_trainable = [v for v in all_trainable if 'fc' in v.name]
else:
net = ResNet_segmentation(self.image_batch, self.conf.num_classes,
True, self.conf.encoder_name)
# Variables that load from pre-trained model.
restore_var = [
v for v in tf.global_variables() if 'resnet_v1' in v.name
]
# Trainable Variables
all_trainable = tf.trainable_variables()
# Fine-tune part
encoder_trainable = [
v for v in all_trainable if 'resnet_v1' in v.name
] # lr * 1.0
# Decoder part
decoder_trainable = [
v for v in all_trainable if 'decoder' in v.name
]
decoder_w_trainable = [
v for v in decoder_trainable
if 'weights' in v.name or 'gamma' in v.name
] # lr * 10.0
decoder_b_trainable = [
v for v in decoder_trainable
if 'biases' in v.name or 'beta' in v.name
] # lr * 20.0
# Check
assert (len(all_trainable) == len(decoder_trainable) +
len(encoder_trainable))
assert (len(decoder_trainable) == len(decoder_w_trainable) +
len(decoder_b_trainable))
# Network raw output
raw_output = net.outputs # [batch_size, h, w, 21]
# Output size
output_shape = tf.shape(raw_output)
output_size = (output_shape[1], output_shape[2])
# Groud Truth: ignoring all labels greater or equal than n_classes
label_proc = prepare_label(self.label_batch,
output_size,
num_classes=self.conf.num_classes,
one_hot=False)
raw_gt = tf.reshape(label_proc, [
-1,
])
indices = tf.squeeze(
tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
raw_prediction = tf.reshape(raw_output, [-1, self.conf.num_classes])
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise softmax_cross_entropy loss
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction, labels=gt)
# L2 regularization
l2_losses = [
self.conf.weight_decay * tf.nn.l2_loss(v) for v in all_trainable
if 'weights' in v.name
]
# Loss function
self.reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
# Define optimizers
# 'poly' learning rate
base_lr = tf.constant(self.conf.learning_rate)
self.curr_step = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(
base_lr,
tf.pow((1 - self.curr_step / self.conf.num_steps),
self.conf.power))
# We have several optimizers here in order to handle the different lr_mult
# which is a kind of parameters in Caffe. This controls the actual lr for each
# layer.
opt_encoder = tf.train.MomentumOptimizer(learning_rate,
self.conf.momentum)
opt_decoder_w = tf.train.MomentumOptimizer(learning_rate * 10.0,
self.conf.momentum)
opt_decoder_b = tf.train.MomentumOptimizer(learning_rate * 20.0,
self.conf.momentum)
# To make sure each layer gets updated by different lr's, we do not use 'minimize' here.
# Instead, we separate the steps compute_grads+update_params.
# Compute grads
grads = tf.gradients(
self.reduced_loss,
encoder_trainable + decoder_w_trainable + decoder_b_trainable)
grads_encoder = grads[:len(encoder_trainable)]
grads_decoder_w = grads[len(encoder_trainable):(
len(encoder_trainable) + len(decoder_w_trainable))]
grads_decoder_b = grads[(len(encoder_trainable) +
len(decoder_w_trainable)):]
# Update params
train_op_conv = opt_encoder.apply_gradients(
zip(grads_encoder, encoder_trainable))
train_op_fc_w = opt_decoder_w.apply_gradients(
zip(grads_decoder_w, decoder_w_trainable))
train_op_fc_b = opt_decoder_b.apply_gradients(
zip(grads_decoder_b, decoder_b_trainable))
# Finally, get the train_op!
update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS
) # for collecting moving_mean and moving_variance
with tf.control_dependencies(update_ops):
self.train_op = tf.group(train_op_conv, train_op_fc_w,
train_op_fc_b)
# Saver for storing checkpoints of the model
self.saver = tf.train.Saver(var_list=tf.global_variables(),
max_to_keep=0)
# Loader for loading the pre-trained model
self.loader = tf.train.Saver(var_list=restore_var)
# Training summary
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, input_size)
raw_output_up = tf.argmax(raw_output_up, axis=3)
self.pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[self.image_batch, 2, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(
decode_labels, [self.label_batch, 2, self.conf.num_classes],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[self.pred, 2, self.conf.num_classes],
tf.uint8)
self.total_summary = tf.summary.image(
'images',
tf.concat(axis=2,
values=[images_summary, labels_summary, preds_summary]),
max_outputs=2) # Concatenate row-wise.
if not os.path.exists(self.conf.logdir):
os.makedirs(self.conf.logdir)
self.summary_writer = tf.summary.FileWriter(
self.conf.logdir, graph=tf.get_default_graph())
def timeflow(self, t=0.0):
wbs = tf.matmul(self.body.wb, self.body.Q) #[1,3] matrix
tot_lbtomots = []
Feqc = tf.constant([0, 0, -Mtot * 9.81], dtype=tf.float32)
Teqc = tf.zeros([1, 3], dtype=tf.float32)
#List of External Forces
Flist = []
llist = []
for p in range(numLeg):
for i in range(numsubleg):
#print('alpha = ',self.leg[p].sub[i].alpha);
self.leg[p].sub[i].omega += self.leg[p].sub[
i].alpha * dtime #omega를 시간에 따라 갱신
#print('omega = ',self.leg[p].sub[i].omega);
self.leg[p].sub[i].theta += self.leg[p].sub[
i].omega * dtime #theta를 시간에 따라 갱신
#print('theta = ',self.leg[p].sub[i].theta);
self.leg[p].sub[i].Q = tf.scalar_mul(tf.cos(self.leg[p].sub[i].theta), tf.eye(3, dtype=tf.float32)) + \
tf.scalar_mul(1.-tf.cos(self.leg[p].sub[i].theta), tf.matmul(self.leg[p].sub[i].axis, self.leg[p].sub[i].axis, transpose_a = True)) + \
tf.scalar_mul(tf.sin(self.leg[p].sub[i].theta), tf.cross(tf.tile(self.leg[p].sub[i].axis,[3,1]), tf.eye(3, dtype=tf.float32)))
Qs = [tf.matmul(self.leg[p].sub[0].Q,
self.body.Q)] #Qs는 i번째 subleg에서 space로의 좌표변환
#List of rotation matrices of each sublegs in space frame
#Type : list of [3,3] Tensor
for i in range(1, numsubleg):
Qs.append(tf.matmul(self.leg[p].sub[i].Q, Qs[i - 1]))
e = [
tf.matmul(self.leg[p].sub[i].axis, Qs[i])
for i in range(numsubleg)
]
#List of axes of each sublegs in space frame
#Type : list of [None,3] Tensor
Qw = [
tf.scalar_mul(self.leg[p].sub[i].omega, e[i])
for i in range(numsubleg)
]
ws = [wbs + Qw[0]]
for i in range(1, numsubleg):
ws.append(ws[i - 1] + Qw[i])
ls = [[
tf.matmul(self.leg[p].sub[i].l[0], Qs[i]),
tf.matmul(self.leg[p].sub[i].l[1], Qs[i])
] for i in range(numsubleg)] #ls = 2Dtensor
lbtomotbs = tf.matmul(self.body.lbtomot[p],
self.body.Q) # lbtomotbs = 2Dtensor
lbtomots = [lbtomotbs + ls[0][0]] # lbtomots = 2Dtensor
for i in range(1, numsubleg):
lbtomots.append(lbtomots[i - 1] + ls[i - 1][1] + ls[i][0])
#Calculating External Forces
vstmp = self.body.vs + tf.cross(wbs, lbtomotbs)
NormalScale = 300.0
TanhConst = 100.0
Zfilter = tf.constant([[0., 0., 1.]])
negZfilter = tf.constant([[0., 0., -1.]])
XYfilter = tf.constant([[1., 1., 0.]])
for i in range(numsubleg):
Fz_primi = tf.multiply(negZfilter,
self.body.rs + lbtomots[i] + ls[i][1])
colbool = tf.reshape(
tf.matmul(Zfilter, tf.nn.relu(Fz_primi), transpose_b=True),
[])
Fz = tf.scalar_mul(NormalScale, tf.nn.relu(Fz_primi))
vstmp += tf.cross(ws[i], ls[i][0] + ls[i][1])
vstmp_z = tf.reshape(
tf.matmul(Zfilter, vstmp, transpose_b=True), [])
Vscale = 3. - tf.nn.tanh(TanhConst * vstmp_z)
Fnormal = tf.scalar_mul(Vscale, Fz)
Flist.append(Fnormal)
vstmp_xy = tf.multiply(XYfilter, vstmp)
Ffric = tf.scalar_mul(-Fricscale,
tf.scalar_mul(colbool, vstmp_xy))
Flist.append(Ffric)
Feqc += Fnormal
Feqc += Ffric
Teqc += tf.cross(lbtomots[i] + ls[i][1], Fnormal + Ffric)
llist.append(tf.norm(lbtomots[i] + ls[i][1]))
#Teqc+=
tot_lbtomots += lbtomots
asb = tf.scalar_mul(Mtotinv, Feqc)
alphab = tf.matmul(tf.matmul(Teqc, self.body.Q, transpose_b=True),
Ibinv)
self.body.wb += tf.scalar_mul(dtime, alphab)
self.body.Q += tf.scalar_mul(
dtime, tf.cross(tf.concat([wbs, wbs, wbs], axis=0), self.body.Q))
self.body.vs += tf.scalar_mul(dtime, asb)
self.body.rs += tf.scalar_mul(dtime, self.body.vs)
# Q to quaternion
'''
qw = tf.scalar_mul(0.5, tf.sqrt(tf.reduce_sum(tf.diag_part(self.body.Q))+1.))
qv = tf.reduce_sum(tf.cross(self.body.Q, tf.eye(3, dtype = tf.float32)), axis = 0)/tf.scalar_mul(4., qw)
# quaternion normalization
qvsquare = tf.reduce_sum(tf.square(qv))
qnorm = tf.sqrt(tf.square(qw)+qvsquare)
qw /= qnorm
qv /= qnorm
# quaternion to Q
self.body.Q = tf.scalar_mul(qw*qw-qvsquare,tf.eye(3, dtype = tf.float32))\
+ 2 * tf.matmul(tf.reshape(qv, [3, 1]), tf.reshape(qv, [1, 3]))\
- 2 * qw * tf.cross(tf.tile(tf.reshape(qv, [1,3]), [3,1]), tf.eye(3, dtype = tf.float32))
'''
return llist, Flist, asb, Qs, [self.body.rs] + [
x + self.body.rs for x in tot_lbtomots
] #, qnorm, qw
def train():
"""Train the Vnet model"""
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
# patch_shape(batch_size, height, width, depth, channels)
input_batch_shape = (FLAGS.batch_size, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1)
output_batch_shape = (FLAGS.batch_size, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1)
images_placeholder, labels_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape)
for batch in range(FLAGS.batch_size):
images_log = tf.cast(images_placeholder[batch:batch+1,:,:,:,0], dtype=tf.uint8)
labels_log = tf.cast(tf.scalar_mul(255,labels_placeholder[batch:batch+1,:,:,:,0]), dtype=tf.uint8)
tf.summary.image("image", tf.transpose(images_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
tf.summary.image("label", tf.transpose(labels_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
'''
import glob
imagelist = glob.glob(os.path.join(FLAGS.data_dir,'Volume','*.nii.gz'))
labellist = [imagename.replace('Volume','Label').replace('volume','label') for imagename in imagelist]
imagetrainlist = imagelist[:int(len(imagelist)/2)]
labeltrainlist = labellist[:int(len(imagelist)/2)]
imagetestlist = imagelist[int(len(imagelist)/2):]
labeltestlist = labellist[int(len(imagelist)/2):]
'''
# Get images and labels
train_data_dir = os.path.join(FLAGS.data_dir,'training')
test_data_dir = os.path.join(FLAGS.data_dir,'testing')
# support multiple image input, but here only use single channel, label file should be a single file with different classes
image_filename = 'image_windowed.nii'
label_filename = 'label.nii'
# image_filename = 'img.nii.gz'
# label_filename = 'label.nii.gz'
# Force input pipepline to CPU:0 to avoid operations sometimes ended up at GPU and resulting a slow down
with tf.device('/cpu:0'):
# create transformations to image and labels
trainTransforms = [
NiftiDataset.Normalization(),
NiftiDataset.Resample(0.2),
NiftiDataset.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
NiftiDataset.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel),
NiftiDataset.RandomNoise()
]
TrainDataset = NiftiDataset.NiftiDataset(
data_dir=train_data_dir,
image_filename=image_filename,
label_filename=label_filename,
transforms=trainTransforms,
train=True
)
trainDataset = TrainDataset.get_dataset()
trainDataset = trainDataset.shuffle(buffer_size=5)
trainDataset = trainDataset.batch(FLAGS.batch_size)
testTransforms = [
NiftiDataset.Normalization(),
NiftiDataset.Resample(0.2),
NiftiDataset.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
NiftiDataset.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel)
]
TestDataset = NiftiDataset.NiftiDataset(
data_dir=test_data_dir,
image_filename=image_filename,
label_filename=label_filename,
transforms=testTransforms,
train=True
)
testDataset = TestDataset.get_dataset()
testDataset = testDataset.shuffle(buffer_size=5)
testDataset = testDataset.batch(FLAGS.batch_size)
train_iterator = trainDataset.make_initializable_iterator()
next_element_train = train_iterator.get_next()
test_iterator = testDataset.make_initializable_iterator()
next_element_test = test_iterator.get_next()
# Initialize the model
with tf.name_scope("vnet"):
logits = VNet.v_net(images_placeholder,input_channels = input_batch_shape[4], output_channels =2)
# # apply weight to the logic funciton
# if FLAGS.class_weight <0 or FLAGS.class_weight>1:
# print("Class weight should between 0 and 1")
# exit()
# if logits.shape[4] == 2:
# class_weight = tf.constant([FLAGS.class_weight,1.0-FLAGS.class_weight])
# weighted_logits = tf.multiply(logits,class_weight)
# else:
# weighted_logits = logits
for batch in range(FLAGS.batch_size):
logits_log_0 = tf.cast(logits[batch:batch+1,:,:,:,0], dtype=tf.uint8)
logits_log_1 = tf.cast(logits[batch:batch+1,:,:,:,1], dtype=tf.uint8)
tf.summary.image("logits_0", tf.transpose(logits_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
tf.summary.image("logits_1", tf.transpose(logits_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# # Exponential decay learning rate
# train_batches_per_epoch = math.ceil(TrainDataset.data_size/FLAGS.batch_size)
# decay_steps = train_batches_per_epoch*FLAGS.decay_steps
with tf.name_scope("learning_rate"):
learning_rate = FLAGS.init_learning_rate
# learning_rate = tf.train.exponential_decay(FLAGS.init_learning_rate,
# global_step,
# decay_steps,
# FLAGS.decay_factor,
# staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
# softmax op for probability layer
with tf.name_scope("softmax"):
softmax_op = tf.nn.softmax(logits,name="softmax")
for batch in range(FLAGS.batch_size):
softmax_log_0 = tf.cast(tf.scalar_mul(255,softmax_op[batch:batch+1,:,:,:,0]), dtype=tf.uint8)
softmax_log_1 = tf.cast(tf.scalar_mul(255,softmax_op[batch:batch+1,:,:,:,1]), dtype=tf.uint8)
tf.summary.image("softmax_0", tf.transpose(softmax_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
tf.summary.image("softmax_1", tf.transpose(softmax_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# Op for calculating loss
with tf.name_scope("cross_entropy"):
loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits,
labels=tf.squeeze(labels_placeholder,
squeeze_dims=[4])))
# loss_op = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(
# logits=logits,
# labels=tf.squeeze(labels_placeholder,squeeze_dims=[4]),
# weights=[[]]))
tf.summary.scalar('loss',loss_op)
# Argmax Op to generate label from logits
with tf.name_scope("predicted_label"):
pred = tf.argmax(logits, axis=4 , name="prediction")
for batch in range(FLAGS.batch_size):
pred_log = tf.cast(tf.scalar_mul(255,pred[batch:batch+1,:,:,:]), dtype=tf.uint8)
tf.summary.image("pred", tf.transpose(pred_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
# Accuracy of model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.expand_dims(pred,-1), tf.cast(labels_placeholder,dtype=tf.int64))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Dice Similarity
with tf.name_scope("dice"):
sorensen = dice_coe(tf.expand_dims(pred,-1),tf.cast(labels_placeholder,dtype=tf.int64), loss_type='sorensen')
jaccard = dice_coe(tf.expand_dims(pred,-1),tf.cast(labels_placeholder,dtype=tf.int64), loss_type='jaccard')
sorensen_loss = 1. - sorensen
jaccard_loss = 1. - jaccard
tf.summary.scalar('sorensen', sorensen)
tf.summary.scalar('jaccard', jaccard)
tf.summary.scalar('sorensen_loss', sorensen_loss)
tf.summary.scalar('jaccard_loss',jaccard_loss)
# Training Op
with tf.name_scope("training"):
# optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=FLAGS.init_learning_rate)
train_op = optimizer.minimize(
loss=loss_op,
global_step=global_step)
# # epoch checkpoint manipulation
start_epoch = tf.get_variable("start_epoch", shape=[1], initializer= tf.zeros_initializer,dtype=tf.int32)
start_epoch_inc = start_epoch.assign(start_epoch+1)
# saver
summary_op = tf.summary.merge_all()
checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir ,"checkpoint")
print("Setting up Saver...")
saver = tf.train.Saver(keep_checkpoint_every_n_hours=5)
# training cycle
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
print("{}: Start training...".format(datetime.datetime.now()))
# summary writer for tensorboard
train_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
test_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)
# restore from checkpoint
if FLAGS.restore_training:
# check if checkpoint exists
if os.path.exists(checkpoint_prefix+"-latest"):
print("{}: Last checkpoint found at {}, loading...".format(datetime.datetime.now(),FLAGS.checkpoint_dir))
latest_checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir,latest_filename="checkpoint-latest")
saver.restore(sess, latest_checkpoint_path)
print("{}: Last checkpoint epoch: {}".format(datetime.datetime.now(),start_epoch.eval()[0]))
print("{}: Last checkpoint global step: {}".format(datetime.datetime.now(),tf.train.global_step(sess, global_step)))
# loop over epochs
for epoch in np.arange(start_epoch.eval(), FLAGS.epochs):
# initialize iterator in each new epoch
sess.run(train_iterator.initializer)
sess.run(test_iterator.initializer)
print("{}: Epoch {} starts".format(datetime.datetime.now(),epoch+1))
# training phase
'''
reader = sitk.ImageFileReader()
for imagepath, labelpath in zip(imagetrainlist,labeltrainlist):
reader.SetFileName(imagepath)
return reader.Execute()
'''
while True:
try:
[image, label] = sess.run(next_element_train)
image = image[:,:,:,:,np.newaxis]
label = label[:,:,:,:,np.newaxis]
train, train_loss, train_accuracy, train_jaccard, summary = sess.run(
[train_op, loss_op, accuracy, jaccard, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label})
print("train loss: %.4f, accuracy: %.4f, jaccard: %.4f" % (train_loss, train_accuracy, train_jaccard))
train_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))
except tf.errors.OutOfRangeError:
start_epoch_inc.op.run()
# print(start_epoch.eval())
# save the model at end of each epoch training
print("{}: Saving checkpoint of epoch {} at {}...".format(datetime.datetime.now(),epoch+1,FLAGS.checkpoint_dir))
saver.save(sess, checkpoint_prefix,
global_step=tf.train.global_step(sess, global_step),
latest_filename="checkpoint-latest")
print("{}: Saving checkpoint succeed".format(datetime.datetime.now()))
break
# testing phase
print("{}: Training of epoch {} finishes, testing start".format(datetime.datetime.now(),epoch+1))
testJaccard = []
while True:
try:
[image, label] = sess.run(next_element_test)
image = image[:,:,:,:,np.newaxis]
label = label[:,:,:,:,np.newaxis]
test_loss, test_accuracy, test_jaccard, summary = sess.run(
[loss_op, accuracy, jaccard, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label})
print("test loss: %.4f, accuracy: %.4f, jaccard: %.4f" % (test_loss, test_accuracy, test_jaccard))
testJaccard.append(test_jaccard)
#loss, summary = sess.run([loss_op, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label})
test_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))
except tf.errors.OutOfRangeError:
print("mean testing Jaccard: {}".format(np.array(testJaccard).mean()))
break
# close tensorboard summary writer
train_summary_writer.close()
test_summary_writer.close()
def apply_bernoulli_dropout(): if scale_during_training: return tf.nn.dropout(inference, keep_prob) else: return tf.scalar_mul(keep_prob,tf.nn.dropout(inference, keep_prob))
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
if args.center_crop_size is None:
center_crop_size = None
else:
hc, wc = map(int, args.center_crop_size.split(','))
center_crop_size = (hc, wc)
with tf.name_scope("create_inputs"):
reader = ImageReader(
DATA_DIR,
DATA_LIST_PATH,
input_size,
center_crop_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN)
image_batch, label_batch = reader.dequeue(args.batch_size)
net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale)
sub4_recls, sub24_recls, sub124_recls = bn_common.extend_reclassifier(net)
restore_var = tf.global_variables()
all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma]
loss_sub4 = create_loss(sub4_recls, label_batch, args)
loss_sub24 = create_loss(sub24_recls, label_batch, args)
loss_sub124 = create_loss(sub124_recls, label_batch, args)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables()
if ('weights' in v.name) or ('kernel' in v.name)]
reduced_loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses)
# print(tf.get_variable_scope().name)
# print(','.join([v.__op.original_name_scope for v in l2_losses]))
# print(','.join([v for v in tf.trainable_variables() if ('beta' in v.name or 'gamma' in v.name)]))
# tf.summary.FileWriter('./summary', tf.get_default_graph())
# exit(0)
# Using Poly learning rate policy
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
# Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
if args.update_mean_var == False:
update_ops = None
else:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
grads = tf.gradients(reduced_loss, all_trainable)
train_op = opt_conv.apply_gradients(zip(grads, all_trainable))
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=99)
ckpt = tf.train.get_checkpoint_state(args.snapshot_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('Restore from pre-trained model...')
net.load(args.restore_from, sess)
# Start queue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
if step % args.save_pred_every == 0:
loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration))
coord.request_stop()
coord.join(threads)
sess.close()
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc_out']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [-1,])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)
total_summary = tf.summary.image('images',
tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
ckpt = tf.train.get_checkpoint_state(SNAPSHOT_DIR)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found.')
load_step = 0
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = { step_ph : step }
if step % args.save_pred_every == 0:
loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, total_summary, train_op], feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def define(self):
config = self.config
with tf.Graph().as_default() as graph:
self.graph = graph
weights, biases = self.make_weights()
self.training = tf.placeholder(tf.bool)
# tf Graph Input
self.X = X = tf.placeholder(
tf.float32, [None, config['height'], config['width']])
if config['conv']:
X1 = conv_net(X, config, weights['conv'], biases['conv'],
config['conv']['max_pool_factors'])
logits = fully_connected(X1, config, weights['fc'],
biases['fc'], self.training)
else:
logits = fully_connected(X, config, weights['fc'],
biases['fc'], self.training)
n_classes = config['n_classes']
self.Y = Y = tf.placeholder(tf.int32, [None, config['n_outputs']])
Yhot = tf.one_hot(Y, n_classes)
logits = tf.reshape(logits, [-1, config['n_outputs'], n_classes])
logit_loss = self.logit_loss(logits, Yhot)
if config['l2_loss']:
if config['conv']:
l2_loss = tf.add_n(
map(lambda weight: tf.nn.l2_loss(weight),
weights['conv']))
else:
l2_loss = tf.constant(0.)
fc_l2_loss = tf.add_n(
map(lambda weight: tf.nn.l2_loss(weight), weights['fc']))
l2_loss = tf.scalar_mul(config['l2_loss']['beta'],
tf.add(l2_loss, fc_l2_loss))
else:
l2_loss = tf.constant(0.)
self.cost = cost = logit_loss #tf.add(logit_loss, l2_loss)
tf.summary.scalar('cost', cost)
self.pred = pred = tf.argmax(logits, axis=2)
correct = tf.cast(tf.equal(tf.cast(pred, tf.int32), Y), tf.float32)
correct_sequence = tf.cast(
tf.equal(tf.reduce_sum(correct, 1), config['n_outputs']),
tf.float32)
self.accuracy = accuracy = tf.reduce_mean(correct_sequence)
self.element_accuracy = element_accuracy = tf.reduce_mean(correct)
tf.summary.scalar('set_accuracy', accuracy)
tf.summary.scalar('element_accuracy', element_accuracy)
self.global_step = global_step = tf.placeholder(tf.int32)
if config['learning_rate']['decay']:
learning_rate = tf.train.exponential_decay(
config['learning_rate']['start'], global_step,
config['iterations'],
config['learning_rate']['coefficient'])
else:
learning_rate = tf.constant(config['learning_rate']['start'])
tf.summary.scalar('learning_rate', learning_rate)
if config['optimizer'] == 'Adam':
self.optimizer = optimizer = tf.train.AdamOptimizer(
learning_rate).minimize(logit_loss)
elif config['optimizer'] == 'GradientDescent':
self.optimizer = optimizer = tf.train.GradientDescentOptimizer(
learning_rate).minimize(cost)
# Initializing the variables
self.init = tf.global_variables_initializer()
self.summary = tf.summary.merge_all()
dirname = 'output/%s_%s' % (
config['exp_name'],
datetime.datetime.now().strftime('%Y_%m_%d_%H.%M'))
self.train_writer = tf.summary.FileWriter(dirname, graph)
# save hype for later comparisons
with open(dirname + '/hypes.json', 'w') as f:
json.dump(config, f, indent=2)
def compute_gradients(self, loss, var_list=None,
gate_gradients=Optimizer.GATE_OP,
aggregation_method=None,
colocate_gradients_with_ops=False,
grad_loss=None):
"""Compute gradients of `loss` for the variables in `var_list`.
This is the first part of `minimize()`. It returns a list
of (gradient, variable) pairs where "gradient" is the gradient
for "variable". Note that "gradient" can be a `Tensor`, an
`IndexedSlices`, or `None` if there is no gradient for the
given variable.
Args:
loss: A Tensor containing the value to minimize or a callable taking
no arguments which returns the value to minimize. When eager execution
is enabled it must be a callable.
var_list: Optional list or tuple of `tf.Variable` to update to minimize
`loss`. Defaults to the list of variables collected in the graph
under the key `GraphKeys.TRAINABLE_VARIABLES`.
gate_gradients: How to gate the computation of gradients. Can be
`GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`.
aggregation_method: Specifies the method used to combine gradient terms.
Valid values are defined in the class `AggregationMethod`.
colocate_gradients_with_ops: If True, try colocating gradients with
the corresponding op.
grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.
Returns:
A list of (gradient, variable) pairs. Variable is always present, but
gradient can be `None`.
Raises:
TypeError: If `var_list` contains anything else than `Variable` objects.
ValueError: If some arguments are invalid.
RuntimeError: If called with eager execution enabled and `loss` is
not callable.
@compatibility(eager)
When eager execution is enabled, `gate_gradients`, `aggregation_method`,
and `colocate_gradients_with_ops` are ignored.
@end_compatibility
"""
if callable(loss):
with backprop.GradientTape() as tape:
if var_list is not None:
tape.watch(var_list)
loss_value = loss()
if var_list is None:
var_list = tape.watched_variables()
# TODO(jhseu): Figure out why GradientTape's gradients don't require loss
# to be executed.
with ops.control_dependencies([loss_value]):
grads = tape.gradient(loss_value, var_list, grad_loss)
return list(zip(grads, var_list))
# # Non-callable/Tensor loss case
# if context.executing_eagerly():
# raise RuntimeError(
# "`loss` passed to Optimizer.compute_gradients should "
# "be a function when eager execution is enabled.")
if gate_gradients not in [SPSA.GATE_NONE, SPSA.GATE_OP, SPSA.GATE_GRAPH]:
raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, "
"Optimizer.GATE_OP, Optimizer.GATE_GRAPH. Not %s" %
gate_gradients)
self._assert_valid_dtypes([loss])
if grad_loss is not None:
self._assert_valid_dtypes([grad_loss])
if var_list is None:
var_list = (
variables.trainable_variables() +
ops.get_collection(ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))
else:
var_list = nest.flatten(var_list)
# pylint: disable=protected-access
var_list += ops.get_collection(ops.GraphKeys._STREAMING_MODEL_PORTS)
# pylint: enable=protected-access
processors = [_get_processor(v) for v in var_list]
if not var_list:
raise ValueError("No variables to optimize.")
var_refs = [p.target() for p in processors]
# print("var_refs:")
# for vr in var_refs:
# print(vr)
# ==================================================================================
# grads = gradients.gradients(
# loss, var_refs, grad_ys=grad_loss,
# gate_gradients=(gate_gradients == SPSA.GATE_OP),
# aggregation_method=aggregation_method,
# colocate_gradients_with_ops=colocate_gradients_with_ops)
# grads = [ tf.zeros(tf.shape(vrefs)) for vrefs in var_refs ]
orig_graph_view = None
trainable_vars = var_list
# self.tvars = var_list
self.tvars = [var.name.split(':')[0] for var in var_list] # list of names of trainable variables
self.global_step_tensor = tf.Variable(0, name='global_step', trainable=False)
# Perturbations
deltas = {}
n_perturbations = {}
p_perturbations = {}
with tf.name_scope("Perturbator"):
self.c_t = tf.div( self.c, tf.pow(tf.add(tf.cast(self.global_step_tensor, tf.float32),
tf.constant(1, dtype=tf.float32)), self.gamma), name = "SPSA_ct" )
for var in trainable_vars:
self.num_params += self._mul_dims(var.get_shape())
var_name = var.name.split(':')[0]
random = Bernoulli(tf.fill(var.get_shape(), 0.5), dtype=tf.float32)
deltas[var] = tf.subtract( tf.constant(1, dtype=tf.float32),
tf.scalar_mul(tf.constant(2, dtype=tf.float32),random.sample(1)[0]), name = "SPSA_delta" )
c_t_delta = tf.scalar_mul( tf.reshape(self.c_t, []), deltas[var] )
n_perturbations[var_name+'/read:0'] = tf.subtract( var, c_t_delta, name = "perturb_n" )
p_perturbations[var_name+'/read:0'] = tf.add(var, c_t_delta, name = "perturb_p" )
# print("{} parameters".format(self.num_params))
# Evaluator
with tf.name_scope("Evaluator"):
orig_graph_view = ge.sgv(tf.get_default_graph())
_, self.ninfo = self._clone_model(orig_graph_view, n_perturbations, 'N_Eval')
_, self.pinfo = self._clone_model(orig_graph_view, p_perturbations, 'P_Eval')
# Weight Updater
optimizer_ops = []
grads = []
with tf.control_dependencies([loss]):
with tf.name_scope('Updater'):
a_t = self.a / (tf.pow(tf.add(tf.cast(self.global_step_tensor, tf.float32),
tf.constant(1, dtype=tf.float32)), self.alpha))
for var in trainable_vars:
l_pos = self.pinfo.transformed( loss )
l_neg = self.ninfo.transformed( loss )
ghat = (l_pos - l_neg) / (tf.constant(2, dtype=tf.float32) * self.c_t * deltas[var])
optimizer_ops.append(tf.assign_sub(var, a_t*ghat))
grads.append(ghat)
print(tf.get_default_graph())
print("grads")
for g in grads:
print(g)
#===================================================================================
if gate_gradients == SPSA.GATE_GRAPH:
print("===================")
grads = control_flow_ops.tuple(grads)
grads_and_vars = list(zip(grads, var_list))
self._assert_valid_dtypes(
[v for g, v in grads_and_vars
if g is not None and v.dtype != dtypes.resource])
return grads_and_vars
def forward(self, reshaped_input):
cluster_weights = tf.get_variable(
"cluster_weights", [self.feature_size, self.cluster_size],
initializer=tf.random_normal_initializer(
stddev=1 / math.sqrt(self.feature_size)))
covar_weights = tf.get_variable(
"covar_weights", [self.feature_size, self.cluster_size],
initializer=tf.random_normal_initializer(
mean=1.0, stddev=1 / math.sqrt(self.feature_size)))
covar_weights = tf.square(covar_weights)
eps = tf.constant([1e-6])
covar_weights = tf.add(covar_weights, eps)
tf.summary.histogram("cluster_weights", cluster_weights)
activation = tf.matmul(reshaped_input, cluster_weights)
if self.add_batch_norm:
activation = slim.batch_norm(activation,
center=True,
scale=True,
is_training=self.is_training,
scope="cluster_bn")
else:
cluster_biases = tf.get_variable(
"cluster_biases", [self.cluster_size],
initializer=tf.random_normal(stddev=1 /
math.sqrt(self.feature_size)))
tf.summary.histogram("cluster_biases", cluster_biases)
activation += cluster_biases
activation = tf.nn.softmax(activation)
tf.summary.histogram("cluster_output", activation)
activation = tf.reshape(activation,
[-1, self.max_frames, self.cluster_size])
a_sum = tf.reduce_sum(activation, -2, keep_dims=True)
if not FLAGS.fv_couple_weights:
cluster_weights2 = tf.get_variable(
"cluster_weights2", [1, self.feature_size, self.cluster_size],
initializer=tf.random_normal_initializer(
stddev=1 / math.sqrt(self.feature_size)))
else:
cluster_weights2 = tf.scalar_mul(FLAGS.fv_coupling_factor,
cluster_weights)
a = tf.multiply(a_sum, cluster_weights2)
activation = tf.transpose(activation, perm=[0, 2, 1])
reshaped_input = tf.reshape(reshaped_input,
[-1, self.max_frames, self.feature_size])
fv1 = tf.matmul(activation, reshaped_input)
fv1 = tf.transpose(fv1, perm=[0, 2, 1])
# computing second order FV
a2 = tf.multiply(a_sum, tf.square(cluster_weights2))
b2 = tf.multiply(fv1, cluster_weights2)
fv2 = tf.matmul(activation, tf.square(reshaped_input))
fv2 = tf.transpose(fv2, perm=[0, 2, 1])
fv2 = tf.add_n([a2, fv2, tf.scalar_mul(-2, b2)])
fv2 = tf.divide(fv2, tf.square(covar_weights))
fv2 = tf.subtract(fv2, a_sum)
fv2 = tf.reshape(fv2, [-1, self.cluster_size * self.feature_size])
fv2 = tf.nn.l2_normalize(fv2, 1)
fv2 = tf.reshape(fv2, [-1, self.cluster_size * self.feature_size])
fv2 = tf.nn.l2_normalize(fv2, 1)
fv1 = tf.subtract(fv1, a)
fv1 = tf.divide(fv1, covar_weights)
fv1 = tf.nn.l2_normalize(fv1, 1)
fv1 = tf.reshape(fv1, [-1, self.cluster_size * self.feature_size])
fv1 = tf.nn.l2_normalize(fv1, 1)
return tf.concat([fv1, fv2], 1)
def main(parsed_arguments):
tf.enable_eager_execution()
if not isinstance(parsed_arguments.threads, int):
parsed_arguments.threads = int(parsed_arguments.threads)
try:
architecture_json_filename = parsed_arguments.json_filename
architecture_json_content = open(architecture_json_filename,
'r').read()
parsed_architecture_json = json.loads(architecture_json_content)
except:
print('Expected a valid architecture json file.')
assert os.path.isdir(parsed_arguments.input)
if not isinstance(parsed_arguments.tile_size, int):
parsed_arguments.tile_size = int(parsed_arguments.tile_size)
if not isinstance(parsed_arguments.tile_overlap_size, int):
parsed_arguments.tile_overlap_size = int(
parsed_arguments.tile_overlap_size)
tile_size = parsed_arguments.tile_size
tile_overlap_size = parsed_arguments.tile_overlap_size
data_format = parsed_arguments.data_format
architecture = Architecture(parsed_architecture_json,
source_data_format='channels_last',
data_format=data_format)
if architecture.data_format == 'channels_first':
use_CPU_only = False
else:
use_CPU_only = True
height = None
width = None
exr_files = OpenEXRDirectory._exr_files(parsed_arguments.input)
features = {}
required_features = architecture.auxiliary_features + architecture.feature_predictions
for feature_prediction in required_features:
exr_loaded = False
if feature_prediction.load_data:
for exr_file in exr_files:
if feature_prediction.name in exr_file:
image = OpenEXRDirectory._load_exr(exr_file)
# HACK: Assume just one source input!
features[Naming.source_feature_name(
feature_prediction.name, index=0)] = image
exr_loaded = True
if height == None:
height = image.shape[0]
width = image.shape[1]
else:
assert height == image.shape[0]
assert width == image.shape[1]
break
else:
image = tf.ones(
[height, width, feature_prediction.number_of_channels])
if feature_prediction.feature_prediction_type != FeaturePredictionType.COLOR:
# Direct and indirect need to be 0.5.
image = tf.scalar_mul(0.5, image)
features[Naming.source_feature_name(feature_prediction.name,
index=0)] = image
exr_loaded = True
if not exr_loaded:
# TODO: Improve (DeepBlender)
raise Exception('Image for \'' + feature_prediction.name +
'\' could not be loaded or does not exist.')
smaller_side_length = min(height, width)
if smaller_side_length < 16:
raise Exception(
'The image needs to have at least a side length of 16 pixels.')
if smaller_side_length < tile_size:
ratio = tile_overlap_size / tile_size
tile_size = smaller_side_length
tile_overlap_size = int(tile_size * ratio)
# Split the images into tiles.
iteration_delta = tile_size - (2 * tile_overlap_size)
width_count = width - (2 * tile_overlap_size) - (2 * iteration_delta)
width_count = width_count / iteration_delta
width_count = math.ceil(width_count) + 2
height_count = height - (2 * tile_overlap_size) - (2 * iteration_delta)
height_count = height_count / iteration_delta
height_count = math.ceil(height_count) + 2
tiled_features_grid = [[None for _ in range(width_count)]
for _ in range(height_count)]
for height_index in range(height_count):
if height_index == 0:
lower_height = 0
upper_height = tile_size
elif height_index == height_count - 1:
upper_height = height
lower_height = upper_height - tile_size
else:
lower_height = height_index * iteration_delta
upper_height = lower_height + tile_size
for width_index in range(width_count):
if width_index == 0:
lower_width = 0
upper_width = tile_size
elif width_index == width_count - 1:
upper_width = width
lower_width = upper_width - tile_size
else:
lower_width = width_index * iteration_delta
upper_width = lower_width + tile_size
tiled_features = {}
for feature_name in features:
feature = features[feature_name]
tiled_feature = feature[lower_height:upper_height,
lower_width:upper_width]
tiled_features[feature_name] = tiled_feature
tiled_features_grid[height_index][width_index] = tiled_features
# We don't need the features anymore.
features = None
if use_CPU_only:
session_config = tf.ConfigProto(device_count={'GPU': 0})
else:
session_config = tf.ConfigProto()
use_XLA = True
if use_XLA:
session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
run_config = tf.estimator.RunConfig(session_config=session_config)
estimator = tf.estimator.Estimator(model_fn=model_fn,
model_dir=architecture.model_directory,
config=run_config,
params={'architecture': architecture})
tiled_features_list = []
for height_index in range(height_count):
for width_index in range(width_count):
tiled_features = tiled_features_grid[height_index][width_index]
tiled_features_list.append(tiled_features)
predictions = estimator.predict(input_fn=lambda: input_fn_predict(
tiled_features_list, tile_size, tile_size))
for height_index in range(height_count):
for width_index in range(width_count):
tiled_features_grid[height_index][width_index] = next(predictions)
predictions = {}
for feature_prediction_tuple in architecture.feature_prediction_tuples:
for feature_prediction in feature_prediction_tuple.feature_predictions:
if feature_prediction.load_data:
horizontal_feature_stripes = []
for height_index in range(height_count):
horizontal_feature_elements = []
for width_index in range(width_count):
tiled_predictions = tiled_features_grid[height_index][
width_index]
prediction_name = Naming.feature_prediction_name(
feature_prediction.name)
prediction = tiled_predictions[prediction_name]
lower_height = 0
upper_height = tile_size
lower_width = 0
upper_width = tile_size
if width_index != 0 and width_index != width_count - 1:
lower_width = tile_overlap_size
upper_width = upper_width - tile_overlap_size
elif width_index == 0 and width_index == width_count - 1:
pass
elif width_index == 0:
upper_width = upper_width - tile_overlap_size
else:
assert width_index == width_count - 1
existing_width = tile_overlap_size + (
(width_count - 1) * (tile_size -
(2 * tile_overlap_size)))
remaining_width = width - existing_width
lower_width = upper_width - remaining_width
if height_index != 0 and height_index != height_count - 1:
lower_height = tile_overlap_size
upper_height = upper_height - tile_overlap_size
elif height_index == 0 and height_index == height_count - 1:
pass
elif height_index == 0:
upper_height = upper_height - tile_overlap_size
else:
assert height_index == height_count - 1
existing_height = tile_overlap_size + (
(height_count - 1) * (tile_size -
(2 * tile_overlap_size)))
remaining_height = height - existing_height
lower_height = upper_height - remaining_height
prediction = prediction[lower_height:upper_height,
lower_width:upper_width]
horizontal_feature_elements.append(prediction)
if len(horizontal_feature_elements) > 1:
horizontal_feature_stripe = np.concatenate(
horizontal_feature_elements, 1)
else:
horizontal_feature_stripe = horizontal_feature_elements[
0]
horizontal_feature_stripes.append(
horizontal_feature_stripe)
if len(horizontal_feature_stripes) > 1:
prediction = np.concatenate(horizontal_feature_stripes, 0)
else:
prediction = horizontal_feature_stripes[0]
prediction_name = Naming.feature_prediction_name(
feature_prediction.name)
predictions[prediction_name] = prediction
diffuse_direct = predictions[Naming.feature_prediction_name(
RenderPasses.DIFFUSE_DIRECT)]
diffuse_indirect = predictions[Naming.feature_prediction_name(
RenderPasses.DIFFUSE_INDIRECT)]
diffuse_color = predictions[Naming.feature_prediction_name(
RenderPasses.DIFFUSE_COLOR)]
glossy_direct = predictions[Naming.feature_prediction_name(
RenderPasses.GLOSSY_DIRECT)]
glossy_indirect = predictions[Naming.feature_prediction_name(
RenderPasses.GLOSSY_INDIRECT)]
glossy_color = predictions[Naming.feature_prediction_name(
RenderPasses.GLOSSY_COLOR)]
subsurface_direct = predictions[Naming.feature_prediction_name(
RenderPasses.SUBSURFACE_DIRECT)]
subsurface_indirect = predictions[Naming.feature_prediction_name(
RenderPasses.SUBSURFACE_INDIRECT)]
subsurface_color = predictions[Naming.feature_prediction_name(
RenderPasses.SUBSURFACE_COLOR)]
transmission_direct = predictions[Naming.feature_prediction_name(
RenderPasses.TRANSMISSION_DIRECT)]
transmission_indirect = predictions[Naming.feature_prediction_name(
RenderPasses.TRANSMISSION_INDIRECT)]
transmission_color = predictions[Naming.feature_prediction_name(
RenderPasses.TRANSMISSION_COLOR)]
volume_direct = predictions[Naming.feature_prediction_name(
RenderPasses.VOLUME_DIRECT)]
volume_indirect = predictions[Naming.feature_prediction_name(
RenderPasses.VOLUME_INDIRECT)]
environment = predictions[Naming.feature_prediction_name(
RenderPasses.ENVIRONMENT)]
emission = predictions[Naming.feature_prediction_name(
RenderPasses.EMISSION)]
# Combined features
diffuse = np.multiply(diffuse_color,
np.add(diffuse_direct, diffuse_indirect))
glossy = np.multiply(glossy_color, np.add(glossy_direct, glossy_indirect))
subsurface = np.multiply(subsurface_color,
np.add(subsurface_direct, subsurface_indirect))
transmission = np.multiply(
transmission_color, np.add(transmission_direct, transmission_indirect))
# Combined image
image = np.add(diffuse, glossy)
image = np.add(image, subsurface)
image = np.add(image, transmission)
image = np.add(image, volume_direct)
image = np.add(image, volume_indirect)
image = np.add(image, environment)
image = np.add(image, emission)
# TODO: Alpha currently ignored.
# Store as npy to open in Blender.
np.save(parsed_arguments.input + '/combined.npy', image)
# HACK: Temporary output as png. (DeepBlender)
image = 255. * image
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
cv2.imwrite(parsed_arguments.input + '/combined.png', image,
[int(cv2.IMWRITE_PNG_COMPRESSION), 9])
dense_shape=[2, 4])
mat = tf.sparse.SparseTensor(
indices=[[1, 0], [2, 0], [3, 0], [3, 1], [0, 2], [1, 3]],
values=[0.85 * 0.33, 0.85 * 0.33, 0.85 * 0.33, 0.85, 0.85, 0.85],
dense_shape=[4, 4])
neg_beta = 0.15 / 4
prev_rank = tf.get_variable("prev_rank",
dtype=float,
initializer=tf.constant([[1 / 4]
for _ in range(4)]))
curr_rank = tf.get_variable("curr_rank",
dtype=float,
initializer=tf.constant([[1 / 4]
for _ in range(4)]))
i = 0
with tf.Session() as sess:
sess.run(init_op)
while True:
print(i)
i += 1
prev_rank = curr_rank
curr_rank = tf.add(
tf.sparse.sparse_dense_matmul(mat, prev_rank),
tf.fill(dims=[4, 1],
value=tf.reduce_sum(tf.scalar_mul(neg_beta, prev_rank))))
tf.global_variables_initializer().run()
#print(sess.run(curr_rank))
if sess.run(tf.reduce_sum(tf.abs(curr_rank - prev_rank))) < 0.00001:
print("Here")
print(sess.run(curr_rank))
break
def train():
global_step = tf.Variable(0, name = 'global_step', trainable = False)
train_dir = CURRENT_DIR + '/logs_without_condition/'
data_dir = CURRENT_DIR + '/data/img_align_celeba_tfrecords/'
images = inputs(data_dir, BATCH_SIZE)
z = tf.placeholder(tf.float32, [None, Z_DIM], name='z')
G = generator(z)
_, D_logits = discriminator(images)
samples = sampler(z)
_, D_logits_ = discriminator(G, reuse = True)
d_loss_real = tf.reduce_mean(tf.scalar_mul(-1, D_logits))
d_loss_fake = tf.reduce_mean(D_logits_)
d_loss = d_loss_real + d_loss_fake
g_loss = tf.reduce_mean(tf.scalar_mul(-1, D_logits_))
z_sum = tf.summary.histogram('z', z)
d_sum = tf.summary.histogram('d', D_logits)
d__sum = tf.summary.histogram('d_', D_logits_)
G_sum = tf.summary.image('G', G)
d_loss_real_sum = tf.summary.scalar('d_loss_real', d_loss_real)
d_loss_fake_sum = tf.summary.scalar('d_loss_fake', d_loss_fake)
d_loss_sum = tf.summary.scalar('d_loss', d_loss)
g_loss_sum = tf.summary.scalar('g_loss', g_loss)
g_sum = tf.summary.merge([z_sum, d__sum, G_sum, d_loss_fake_sum, g_loss_sum])
d_sum = tf.summary.merge([z_sum, d_sum, d_loss_real_sum, d_loss_sum])
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]
saver = tf.train.Saver()
with tf.variable_scope("", reuse=tf.AUTO_REUSE):
d_optim = tf.train.RMSPropOptimizer(LR).minimize(d_loss, var_list = d_vars, global_step = global_step)
g_optim = tf.train.RMSPropOptimizer(LR).minimize(g_loss, var_list = g_vars, global_step = global_step)
clip_d_op = [var.assign(tf.clip_by_value(var, CLIP[0], CLIP[1])) for var in d_vars]
#os.environ['CUDA_VISIBLE_DEVICES'] = str(0)
#config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.2
#sess = tf.InteractiveSession(config=config)
sess = tf.Session()
writer = tf.summary.FileWriter(train_dir, sess.graph)
sample_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess = sess, coord = coord)
init = tf.initialize_all_variables()
sess.run(init)
start = 0
if LOAD_MODEL:
print(" [*] Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(train_dir)
if ckpt and ckpt.model_checkpoint_path:
ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
saver.restore(sess, os.path.join(train_dir, ckpt_name))
global_step = ckpt.model_checkpoint_path.split('/')[-1]\
.split('-')[-1]
print('Loading success, global_step is %s' % global_step)
start = int(global_step)
for epoch in range(EPOCH):
batch_idxs = 3072
if epoch:
start = 0
for idx in range(start, batch_idxs):
if idx<25 or idx % 500 == 0:
critic_num = 25
else:
critic_num = CRITIC_NUM
for _ in range(critic_num):
batch_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM))
_,summary_str = sess.run([d_optim,d_sum], feed_dict = {z:batch_z})
sess.run(clip_d_op)
writer.add_summary(summary_str,idx+1)
batch_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM))
_, summary_str = sess.run([d_optim, d_sum], feed_dict = {z: batch_z})
writer.add_summary(summary_str, idx+1)
# Update G network
_, summary_str = sess.run([g_optim, g_sum], feed_dict = {z: batch_z})
writer.add_summary(summary_str, idx+1)
# Run g_optim twice to make sure that d_loss does not go to zero
_, summary_str = sess.run([g_optim, g_sum], feed_dict = {z: batch_z})
writer.add_summary(summary_str, idx+1)
errD_fake = 0#d_loss_fake.eval({z: batch_z})
errD_real = 0#d_loss_real.eval()
errG = 0#g_loss.eval({z: batch_z})
if idx % 20 == 0:
print("[%4d/%4d] d_loss: %.8f, g_loss: %.8f" \
% (idx, batch_idxs, errD_fake+errD_real, errG))
if idx % 100 == 0:
sample = sess.run(samples, feed_dict = {z: sample_z})
samples_path = CURRENT_DIR + '\\out\\'
save_images(sample, [8, 8],
samples_path + \
'sample_%d_epoch_%d.png' % (epoch, idx))
print('\n'*2)
print('=========== %d_epoch_%d.png save down ==========='
%(epoch, idx))
print('\n'*2)
if (idx % 512 == 0) or (idx + 1 == batch_idxs):
checkpoint_path = os.path.join(train_dir,
'my_dcgan_tfrecords.ckpt')
saver.save(sess, checkpoint_path, global_step = idx+1)
print('********* model saved *********')
print('******* start with %d *******' % start)
coord.request_stop()
coord.join(threads)
sess.close()
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_scale, args.random_mirror, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
image_batch075 = tf.image.resize_images(
image_batch, [int(h * 0.75), int(w * 0.75)])
image_batch05 = tf.image.resize_images(
image_batch, [int(h * 0.5), int(w * 0.5)])
# Create network.
with tf.variable_scope('', reuse=False):
net = DeepLabResNetModel({'data': image_batch},
is_training=args.is_training)
with tf.variable_scope('', reuse=True):
net075 = DeepLabResNetModel({'data': image_batch075},
is_training=args.is_training)
with tf.variable_scope('', reuse=True):
net05 = DeepLabResNetModel({'data': image_batch05},
is_training=args.is_training)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output100 = net.layers['fc1_voc12']
raw_output075 = net075.layers['fc1_voc12']
raw_output05 = net05.layers['fc1_voc12']
raw_output = tf.reduce_max(tf.stack([
raw_output100,
tf.image.resize_images(raw_output075,
tf.shape(raw_output100)[1:3, ]),
tf.image.resize_images(raw_output05,
tf.shape(raw_output100)[1:3, ])
]),
axis=0)
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = tf.global_variables()
all_trainable = [
v for v in tf.trainable_variables()
if 'beta' not in v.name and 'gamma' not in v.name
]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable
if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable
if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable
if 'biases' in v.name] # lr * 20.0
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, n_classes])
raw_prediction100 = tf.reshape(raw_output100, [-1, n_classes])
raw_prediction075 = tf.reshape(raw_output075, [-1, n_classes])
raw_prediction05 = tf.reshape(raw_output05, [-1, n_classes])
label_proc = prepare_label(label_batch,
tf.pack(raw_output.get_shape()[1:3]),
one_hot=False) # [batch_size, h, w]
label_proc075 = prepare_label(label_batch,
tf.pack(raw_output075.get_shape()[1:3]),
one_hot=False)
label_proc05 = prepare_label(label_batch,
tf.pack(raw_output05.get_shape()[1:3]),
one_hot=False)
raw_gt = tf.reshape(label_proc, [
-1,
])
raw_gt075 = tf.reshape(label_proc075, [
-1,
])
raw_gt05 = tf.reshape(label_proc05, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, n_classes - 1)), 1)
indices075 = tf.squeeze(tf.where(tf.less_equal(raw_gt075, n_classes - 1)),
1)
indices05 = tf.squeeze(tf.where(tf.less_equal(raw_gt05, n_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32)
gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32)
prediction = tf.gather(raw_prediction, indices)
prediction100 = tf.gather(raw_prediction100, indices)
prediction075 = tf.gather(raw_prediction075, indices075)
prediction05 = tf.gather(raw_prediction05, indices05)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
labels=gt)
loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction100, labels=gt)
loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction075, labels=gt075)
loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction05, labels=gt05)
l2_losses = [
args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'weights' in v.name
]
reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean(
loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(loss05) + tf.add_n(
l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[image_batch, args.save_num_images], tf.uint8)
labels_summary = tf.py_func(decode_labels,
[label_batch, args.save_num_images], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images],
tf.uint8)
total_summary = tf.summary.image(
'images',
tf.concat(2, [images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(
base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
# Define a variable to accumulate gradients.
accum_grads = [
tf.Variable(tf.zeros_like(v.initialized_value()), trainable=False)
for v in conv_trainable + fc_w_trainable + fc_b_trainable
]
# Define an operation to clear the accumulated gradients for next batch.
zero_op = [v.assign(tf.zeros_like(v)) for v in accum_grads]
# Compute gradients.
grads = tf.gradients(reduced_loss,
conv_trainable + fc_w_trainable + fc_b_trainable)
# Accumulate and normalise the gradients.
accum_grads_op = [
accum_grads[i].assign_add(grad / args.grad_update_every)
for i, grad in enumerate(grads)
]
grads_conv = accum_grads[:len(conv_trainable)]
grads_fc_w = accum_grads[len(conv_trainable):(len(conv_trainable) +
len(fc_w_trainable))]
grads_fc_b = accum_grads[(len(conv_trainable) + len(fc_w_trainable)):]
# Apply the gradients.
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=restore_var, max_to_keep=10)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
loss_value = 0
# Clear the accumulated gradients.
sess.run(zero_op, feed_dict=feed_dict)
# Accumulate gradients.
for i in range(args.grad_update_every):
_, l_val = sess.run([accum_grads_op, reduced_loss],
feed_dict=feed_dict)
loss_value += l_val
# Normalise the loss.
loss_value /= args.grad_update_every
# Apply gradients.
if step % args.save_pred_every == 0:
images, labels, summary, _ = sess.run(
[image_batch, label_batch, total_summary, train_op],
feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
sess.run(train_op, feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(
step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def train(target, dataset, cluster_spec):
"""Train Inception on a dataset for a number of steps."""
# Number of workers and parameter servers are inferred from the workers and ps
# hosts string.
num_workers = len(cluster_spec.as_dict()['worker'])
num_parameter_servers = len(cluster_spec.as_dict()['ps'])
# If no value is given, num_replicas_to_aggregate defaults to be the number of
# workers.
if FLAGS.num_replicas_to_aggregate == -1:
num_replicas_to_aggregate = num_workers
else:
num_replicas_to_aggregate = FLAGS.num_replicas_to_aggregate
# Both should be greater than 0 in a distributed training.
assert num_workers > 0 and num_parameter_servers > 0, (' num_workers and '
'num_parameter_servers'
' must be > 0.')
# Choose worker 0 as the chief. Note that any worker could be the chief
# but there should be only one chief.
is_chief = (FLAGS.task_id == 0)
#batchSizeManager = BatchSizeManager(32, 4)
# Ops are assigned to worker by default.
tf.logging.info('cccc-num_parameter_servers:'+str(num_parameter_servers))
partitioner = tf.fixed_size_partitioner(num_parameter_servers, 0)
device_setter = tf.train.replica_device_setter(ps_tasks=num_parameter_servers)
slim = tf.contrib.slim
with tf.device('/job:worker/task:%d' % FLAGS.task_id):
with tf.variable_scope('root', partitioner=partitioner):
# Variables and its related init/assign ops are assigned to ps.
# with slim.arg_scope(
# [slim.variables.variable, slim.variables.global_step],
# device=slim.variables.VariableDeviceChooser(num_parameter_servers)):
with tf.device(device_setter):
# partitioner=partitioner):
# Create a variable to count the number of train() calls. This equals the
# number of updates applied to the variables.
# global_step = slim.variables.global_step()
global_step = tf.Variable(0, trainable=False)
# Calculate the learning rate schedule.
batch_size = tf.placeholder(dtype=tf.int32, shape=(), name='batch_size')
num_batches_per_epoch = (dataset.num_examples_per_epoch() /
FLAGS.batch_size)
# Decay steps need to be divided by the number of replicas to aggregate.
decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay /
num_replicas_to_aggregate)
# Decay the learning rate exponentially based on the number of steps.
lr = tf.train.exponential_decay(FLAGS.initial_learning_rate,
global_step,
decay_steps,
FLAGS.learning_rate_decay_factor,
staircase=True)
# Add a summary to track the learning rate.
# tf.summary.scalar('learning_rate', lr)
# Create an optimizer that performs gradient descent.
opt = tf.train.RMSPropOptimizer(lr,
RMSPROP_DECAY,
momentum=RMSPROP_MOMENTUM,
epsilon=RMSPROP_EPSILON)
images, labels = image_processing.distorted_inputs(
dataset,
batch_size,
num_preprocess_threads=FLAGS.num_preprocess_threads)
print(images.get_shape())
print(labels.get_shape())
# Number of classes in the Dataset label set plus 1.
# Label 0 is reserved for an (unused) background class.
# num_classes = dataset.num_classes() + 1
num_classes = dataset.num_classes()
print(num_classes)
# logits = inception.inference(images, num_classes, for_training=True)
network_fn = nets_factory.get_network_fn('inception_v3',num_classes=num_classes)
(logits,_) = network_fn(images)
print(logits.get_shape())
# Add classification loss.
# inception.loss(logits, labels, batch_size)
# Gather all of the losses including regularization losses.
labels = tf.one_hot(labels, 1000, 1, 0)
cross_entropy = tf.losses.softmax_cross_entropy(
logits=logits,
onehot_labels=labels)
# losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
# losses += tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
total_loss = cross_entropy + _WEIGHT_DECAY * tf.add_n(
[tf.nn.l2_loss(v) for v in tf.trainable_variables()])
# total_loss = tf.add_n(losses, name='total_loss')
if is_chief:
# Compute the moving average of all individual losses and the
# total loss.
loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
loss_averages_op = loss_averages.apply(losses + [total_loss])
# Attach a scalar summmary to all individual losses and the total loss;
# do the same for the averaged version of the losses.
# for l in losses + [total_loss]:
# loss_name = l.op.name
# Name each loss as '(raw)' and name the moving average version of the
# loss as the original loss name.
# tf.summary.scalar(loss_name + ' (raw)', l)
# tf.summary.scalar(loss_name, loss_averages.average(l))
# Add dependency to compute loss_averages.
with tf.control_dependencies([loss_averages_op]):
total_loss = tf.identity(total_loss)
# Track the moving averages of all trainable variables.
# Note that we maintain a 'double-average' of the BatchNormalization
# global statistics.
# This is not needed when the number of replicas are small but important
# for synchronous distributed training with tens of workers/replicas.
exp_moving_averager = tf.train.ExponentialMovingAverage(
MOVING_AVERAGE_DECAY, global_step)
variables_to_average = (
tf.trainable_variables() + tf.moving_average_variables())
# Add histograms for model variables.
# for var in variables_to_average:
# tf.summary.histogram(var.op.name, var)
# Create synchronous replica optimizer.
opt = tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=num_replicas_to_aggregate,
total_num_replicas=num_workers,
variable_averages=exp_moving_averager,
variables_to_average=variables_to_average)
# batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION)
# assert batchnorm_updates, 'Batchnorm updates are missing'
# batchnorm_updates_op = tf.group(*batchnorm_updates)
# # Add dependency to compute batchnorm_updates.
# with tf.control_dependencies([batchnorm_updates_op]):
# total_loss = tf.identity(total_loss)
# Compute gradients with respect to the loss.
# grads = opt.compute_gradients(total_loss)
grads0 = opt.compute_gradients(total_loss)
grads = [(tf.scalar_mul(tf.cast(batch_size/FLAGS.batch_size, tf.float32), grad), var) for grad, var in grads0]
# Add histograms for gradients.
# for grad, var in grads:
# if grad is not None:
# tf.summary.histogram(var.op.name + '/gradients', grad)
apply_gradients_op = opt.apply_gradients(grads, global_step=global_step)
with tf.control_dependencies([apply_gradients_op]):
train_op = tf.identity(total_loss, name='train_op')
# Get chief queue_runners and init_tokens, which is used to synchronize
# replicas. More details can be found in SyncReplicasOptimizer.
chief_queue_runners = [opt.get_chief_queue_runner()]
init_tokens_op = opt.get_init_tokens_op()
# Create a saver.
saver = tf.train.Saver()
# Build the summary operation based on the TF collection of Summaries.
# summary_op = tf.summary.merge_all()
# Build an initialization operation to run below.
init_op = tf.global_variables_initializer()
# We run the summaries in the same thread as the training operations by
# passing in None for summary_op to avoid a summary_thread being started.
# Running summaries and training operations in parallel could run out of
# GPU memory.
sv = tf.train.Supervisor(is_chief=is_chief,
logdir=FLAGS.train_dir,
init_op=init_op,
summary_op=None,
global_step=global_step,
recovery_wait_secs=1,
saver=None,
save_model_secs=FLAGS.save_interval_secs)
tf.logging.info('%s Supervisor' % datetime.now())
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=FLAGS.log_device_placement)
# Get a session.
sess = sv.prepare_or_wait_for_session(target, config=sess_config)
# Start the queue runners.
queue_runners = tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS)
sv.start_queue_runners(sess, queue_runners)
tf.logging.info('Started %d queues for processing input data.',
len(queue_runners))
if is_chief:
sv.start_queue_runners(sess, chief_queue_runners)
sess.run(init_tokens_op)
# Train, checking for Nans. Concurrently run the summary operation at a
# specified interval. Note that the summary_op and train_op never run
# simultaneously in order to prevent running out of GPU memory.
# next_summary_time = time.time() + FLAGS.save_summaries_secs
step = 0
time0 = time.time()
batch_size_num = 1
while not sv.should_stop():
try:
start_time = time.time()
batch_size_num = 32
# batch_size_num = int((int(step)/3*10)) % 100000 + 1
# if step < 5:
# batch_size_num = 32
# batch_size_num = (batch_size_num ) % 64 + 1
# else:
# batch_size_num = 80
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
my_images, loss_value, step = sess.run([images, train_op, global_step], feed_dict={batch_size: batch_size_num}, options=run_options, run_metadata=run_metadata)
b = time.time()
# assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
if step > FLAGS.max_steps:
break
duration = time.time() - start_time
thread = threading2.Thread(target=get_computation_time, name="get_computation_time",args=(run_metadata.step_stats,step,))
thread.start()
# tl = timeline.Timeline(run_metadata.step_stats)
# last_batch_time = tl.get_local_step_duration('sync_token_q_Dequeue')
c0 = time.time()
# batch_size_num = batchSizeManager.dictate_new_batch_size(FLAGS.task_id, last_batch_time)
# batch_size_num = rpcClient.update_batch_size(FLAGS.task_id, last_batch_time, available_cpu, available_memory, step, batch_size_num)
# ctf = tl.generate_chrome_trace_format()
# with open("timeline.json", 'a') as f:
# f.write(ctf)
if step % 1 == 0:
examples_per_sec = FLAGS.batch_size / float(duration)
c = time.time()
tf.logging.info("time statistics" + " - train_time: " + str(b-start_time) + " - get_batch_time: " + str(c0-b) + " - get_bs_time: " + str(c-c0) + " - accum_time: " + str(c-time0) + " - batch_size: " + str(batch_size_num))
format_str = ('Worker %d: %s: step %d, loss = %.2f'
'(%.1f examples/sec; %.3f sec/batch)')
tf.logging.info(format_str %
(FLAGS.task_id, datetime.now(), step, loss_value,
examples_per_sec, duration))
# Determine if the summary_op should be run on the chief worker.
# if is_chief and next_summary_time < time.time():
# tf.logging.info('Running Summary operation on the chief.')
# summary_str = sess.run(summary_op)
# sv.summary_computed(sess, summary_str)
# tf.logging.info('Finished running Summary operation.')
# Determine the next time for running the summary.
# next_summary_time += FLAGS.save_summaries_secs
except:
if is_chief:
tf.logging.info('Chief got exception while running!')
raise
# Stop the supervisor. This also waits for service threads to finish.
sv.stop()
def apply_gradients(self, grads_and_vars, worker_id, global_step=None, name=None, collect_cdfs=False,
# batch_idx_list=None, worker_kill_list=None, num_workers=None, num_batches_per_epoch=None):
matrix_to_solve=None, num_batches_per_epoch=None):
"""Apply gradients to variables.
This contains most of the synchronization implementation and also wraps the
apply_gradients() from the real optimizer.
Args:
grads_and_vars: List of (gradient, variable) pairs as returned by
compute_gradients().
global_step: Optional Variable to increment by one after the
variables have been updated.
name: Optional name for the returned operation. Default to the
name passed to the Optimizer constructor.
Returns:
train_op: The op to dequeue a token so the replicas can exit this batch
and start the next one. This is executed by each replica.
Raises:
ValueError: If the grads_and_vars is empty.
ValueError: If global step is not provided, the staleness cannot be
checked.
"""
if not grads_and_vars:
raise ValueError("Must supply at least one variable")
if global_step is None:
raise ValueError("Global step is required to check staleness")
self._global_step = global_step
train_ops = []
aggregated_grad = []
var_list = []
self._local_step = variables.Variable(
initial_value=0,
trainable=False,
collections=[ops.GraphKeys.LOCAL_VARIABLES],
dtype=global_step.dtype.base_dtype,
name="sync_rep_local_step")
self.local_step_init_op = state_ops.assign(self._local_step, global_step._ref())
chief_init_ops = [self.local_step_init_op]
self.ready_for_local_init_op = variables.report_uninitialized_variables(
variables.all_variables())
# The wait op waits for the current worker to dequeue a token from its respective token queue
self._wait_op = self._sync_token_queues[worker_id].dequeue()
# Replicas have to wait until they can get a token from the token queue
# BEFORE begining to compute gradients.
with ops.device(global_step.device):
queue_size = self._sync_token_queues[worker_id].size()
update_local_step_op = state_ops.assign(self._local_step, global_step._ref())
# Gradient accum creation
with ops.name_scope(None, self._name):
for grad, var in grads_and_vars:
var_list.append(var)
tf.logging.info("Grad " + str(grad) + " assigned to " + str(var.device))
with ops.device(var.device):
if grad is None:
continue
elif isinstance(grad, ops.Tensor):
grad_accum = data_flow_ops.ConditionalAccumulator(
grad.dtype,
shape=var.get_shape(),
shared_name=var.name + "/grad_accum")
else:
if not isinstance(grad, ops.IndexedSlices):
raise ValueError("Unknown grad type!")
grad_accum = data_flow_ops.SparseConditionalAccumulator(
grad.dtype, shape=(), shared_name=var.name + "/grad_accum")
self._accumulator_list.append((grad_accum, var))
"""# Phase 1 gradient computation
with ops.control_dependencies([update_local_step_op]):
for index, (grad, var) in enumerate(grads_and_vars):
with ops.device(var.device):
if grad is None:
continue
elif isinstance(grad, ops.Tensor):
grad_accum = self._accumulator_list[index][0]
train_ops.append(grad_accum.apply_grad(grad,
local_step=self._local_step._ref()))
else:
if not isinstance(grad, ops.IndexedSlices):
raise ValueError("Unknown grad type!")
grad_accum = self._accumulator_list[index][0]
train_ops.append(grad_accum.apply_indexed_slices_grad(
grad, local_step=self._local_step._ref()))"""
# Phase 1 gradient computation
with ops.control_dependencies([update_local_step_op]):
for index, (grad, var) in enumerate(grads_and_vars):
print_start_op = logging_ops.Print(global_step, [global_step], message="Starting to apply grads for variable %d" % index)
train_ops.append(print_start_op)
with ops.device(var.device):
ps_step_printer0 = logging_ops.Print(global_step, [global_step], message="global step printer0 on ps")
train_ops.append(ps_step_printer0)
'''Implement LS computation and solution here'''
#b = np.ones(int(num_batches_per_epoch))
b = tf.ones([int(num_batches_per_epoch),1], tf.float32)
A = matrix_to_solve
# A_for_calc = np.transpose(A)
LS_solution = linalg_ops.matrix_solve_ls(A, b, fast=False)
LS_calc = tf.reshape(LS_solution, [-1])
weight = tf.slice(LS_calc, [worker_id], [1])
# print_ls_op = logging_ops.Print(LS_calc, [LS_calc], message="Solution for LS!")
# train_ops.append(print_ls_op)
weighted_grad = tf.scalar_mul(weight[0], grad)
'''Kill some workers'''
if grad is None:
continue
elif isinstance(grad, ops.Tensor):
grad_accum = self._accumulator_list[index][0]
num_accum = grad_accum.num_accumulated()
tf.logging.info("Grad Accumed %s, Worker ID: %s" % (str(num_accum), str(worker_id)))
with ops.control_dependencies([print_start_op]):
with tf.device("job:worker/task:%d" % worker_id):
apply_grad_op = grad_accum.apply_grad(grad,
# apply_grad_op = grad_accum.apply_grad(weighted_grad,
local_step=self._local_step._ref())
with ops.control_dependencies([apply_grad_op]):
finished_print_op = logging_ops.Print(global_step, [global_step], message="Done applying grads for variable %d" % index)
train_ops.append(finished_print_op)
else:
if not isinstance(grad, ops.IndexedSlices):
raise ValueError("Unknown grad type!")
grad_accum = self._accumulator_list[index][0]
with ops.control_dependencies([print_start_op]):
with tf.device("job:worker/task:%d" % worker_id):
apply_grad_op = grad_accum.apply_indexed_slices_grad(
grad, local_step=self._local_step._ref())
# weighted_grad, local_step=self._local_step._ref())
with ops.control_dependencies([apply_grad_op]):
finished_print_op = logging_ops.Print(global_step, [global_step], message="Done applying grads for variable %d" % index)
train_ops.append(finished_print_op)
# Phase 2 gradient applying
for index, (grad, var) in enumerate(grads_and_vars):
with ops.device(var.device):
work_idx_print1 = logging_ops.Print(worker_id, [worker_id], message="worker id for aggregate grad")
ps_step_printer1 = logging_ops.Print(global_step, [global_step], message="global step printer1 on ps")
num_replica_aggragate = logging_ops.Print(self._replicas_to_aggregate, [self._replicas_to_aggregate], message="num replica aggregate")
train_ops.append(work_idx_print1)
train_ops.append(ps_step_printer1)
train_ops.append(num_replica_aggragate)
grad_accum = self._accumulator_list[index][0]
if grad is None:
aggregated_grad.append(None)
elif isinstance(grad, ops.Tensor):
if collect_cdfs:
# aggregated_grad.append(grad_accum.take_grad(self._total_num_replicas))
aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate))
else:
aggregated_grad.append(grad_accum.take_grad(1))
else:
if collect_cdfs:
# aggregated_grad.append(grad_accum.take_grad(self._total_num_replicas))
aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate))
else:
aggregated_grad.append(grad_accum.take_indexed_slices_grad(1))
aggregated_grads_and_vars = zip(aggregated_grad, var_list)
# Some debug operations
self.print_sizes = logging_ops.Print(global_step, [self._sync_token_queues[i].size() for i in range(self._total_num_replicas)], message="queue sizes")
self.print_accum_sizes = logging_ops.Print(self._local_step,
[x[0].num_accumulated() for x in self._accumulator_list] + [worker_id],
message="Accum sizes")
self.print_local_step = logging_ops.Print(self._local_step, [self._local_step._ref(), global_step._ref()], message="local vs global step")
# sync_op will be assigned to the same device as the global step.
with ops.device(global_step.device), ops.name_scope(""):
with ops.control_dependencies([self.print_accum_sizes]):
update_op = self._opt.apply_gradients(aggregated_grads_and_vars, global_step)
self._update_op = update_op
with ops.control_dependencies([update_op]):
sync_op = []
for cur_worker_id in range(self._total_num_replicas):
sync_op.append(self._sync_token_queues[cur_worker_id].enqueue(global_step))
sync_op = control_flow_ops.group(*(sync_op))
# dummy_queue is passed to the queue runner. Don't use the real queues
# because the queue runner doesn't automatically reopen it once it
# closed queues in PS devices.
dummy_queue = (
data_flow_ops.FIFOQueue(1,
types_pb2.DT_INT32,
shapes=(),
shared_name="dummy_queue"))
self._chief_queue_runner = queue_runner.QueueRunner(dummy_queue,
[sync_op])
with ops.device(global_step.device), ops.name_scope(""):
with ops.control_dependencies(train_ops):
# Worker finished applying gradients. Add token to phase1_finished_queue
train_op = logging_ops.Print(self._local_step._ref(),
[x[0].num_accumulated() for x in self._accumulator_list] + [worker_id],
message="Finished worker updates",
name="FinishedWorkerUpdatesPrint")
for accum, var in self._accumulator_list:
with ops.device(var.device):
chief_init_ops.append(
accum.set_global_step(
global_step, name="SetGlobalStep"))
self.chief_init_op = control_flow_ops.group(*(chief_init_ops))
self._gradients_applied = True
return train_op
def main():
"""Create the model and start the training."""
args = get_arguments()
args.snapshot_dir=args.snapshot_dir.replace('DeepDICD/','DeepDICD/'+args.model_name+'-'+args.domain+'-')
print(toMagenta(args.snapshot_dir))
ss=args.domain.split('-')
if ss[0]=='D':
args.list1=args.list1.replace('amazon.txt','dslr.txt')
elif ss[0]=='W':
args.list1=args.list1.replace('amazon.txt','webcam.txt')
if ss[1]=='A':
args.list2=args.list2.replace('dslr.txt','amazon.txt')
elif ss[1]=='W':
args.list2=args.list2.replace('dslr.txt','webcam.txt')
print(toMagenta(args.list1))
print(toMagenta(args.list2))
start_steps=args.start_steps
h=args.h
w=args.w
# construct data generator
file1 = open(args.list1)
num1 = len(file1.readlines())
file2 = open(args.list2)
num2 = len(file2.readlines())
file1.close()
file2.close()
steps_per_epoch=int((num1/(args.batch_size)))
num_steps=int(steps_per_epoch*args.num_epochs)
val_num_steps=int(num2/args.batch_size)
print(toCyan('src domain: {:d}, tar domain {:d}'.format(num1,num2)))
print(toCyan('steps_per_epoch x num_epochs:{:d} x {:d}'.format(steps_per_epoch,args.num_epochs)))
# Chong
# split_batch_size=int(args.batch_size/hvd.size())
myDataloader=Dataloader(args.img_dir,args.list1,args.list2,args.batch_size,args.h,args.w,args.num_threads)
src_img=myDataloader.simg_batch
src_label=myDataloader.slabel_batch
tar_img=myDataloader.timg_batch
tar_label=myDataloader.tlabel_batch
coord = tf.train.Coordinator()
# Learning Startegy Settings
baseLR1 = tf.constant(args.lr1)
baseLR2 = tf.constant(args.lr2)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
if args.learning_strategy==0:
lr1=baseLR1/tf.pow(1+0.001*step_ph/steps_per_epoch,0.75)
lr2=baseLR2/tf.pow(1+0.001*step_ph/steps_per_epoch,0.75)
elif args.learning_strategy==1:
lr1=baseLR1/tf.pow(1+0.0005*step_ph/steps_per_epoch,0.75)
lr2=baseLR2/tf.pow(1+0.0005*step_ph/steps_per_epoch,0.75)
if args.learning_strategy==2:
lr1=baseLR1/tf.pow(1+0.005*step_ph/steps_per_epoch,0.75)
lr2=baseLR2/tf.pow(1+0.005*step_ph/steps_per_epoch,0.75)
if args.learning_strategy==3:
lr1=baseLR1/tf.pow(1+0.001*step_ph,0.75)
lr2=baseLR2/tf.pow(1+0.001*step_ph,0.75)
# lr1 = tf.scalar_mul(baseLR1, tf.pow((1 - step_ph / num_steps), args.power))
# lr2 = tf.scalar_mul(baseLR2, tf.pow((1 - step_ph / num_steps), args.power))
# lr1=baseLR1/tf.pow(1+0.001*step_ph,0.75)
# lr2=baseLR2/tf.pow(1+0.001*step_ph,0.75)
# Polynomial_decay
# lr1=baseLR1
# lr2=baseLR2
# decay_steps=steps_per_epoch*10
# lr1=tf.train.exponential_decay(baseLR1,step_ph,decay_steps,0.1,staircase=True)
# lr2=tf.train.exponential_decay(baseLR2,step_ph,decay_steps,0.1,staircase=True)
keep_prob=tf.placeholder(dtype=tf.float32,shape=())
p=(1- tf.scalar_mul(1., tf.pow((1 - step_ph / num_steps), args.power)))
alpha1 = adaptation_factor(step_ph*1.0/10000)
# alpha1 = tf.constant(1.,tf.float32)
alpha2 = 0.01*p
# alpha2 = 0
model = DeepDICDModel(args,keep_prob,src_img,src_label,tar_img,tar_label)
model.build_losses()
model.build_outputs()
all_trainable_var=[v for v in tf.trainable_variables()]
fine_tune_var = [v for v in all_trainable_var if 'fc8' not in v.name and 'fc9' not in v.name and 'domain' not in v.name]
fine_tune_var_weights=[v for v in fine_tune_var if 'weights' in v.name]
fine_tune_var_bias=[v for v in fine_tune_var if 'bias' in v.name]
retrain_var = [v for v in all_trainable_var if 'fc8' in v.name or 'fc9' in v.name]
retrain_var_weights=[v for v in retrain_var if 'weights' in v.name]
retrain_var_bias=[v for v in retrain_var if 'bias' in v.name]
domain_var=[v for v in all_trainable_var if 'domain' in v.name]
domain_var_weights=[v for v in domain_var if 'weights' in v.name]
domain_var_bias=[v for v in domain_var if 'bias' in v.name]
print(toGreen(domain_var_bias))
model.build_regular_losses(domain_var_weights,fine_tune_var_weights+fine_tune_var_weights)
summary_=model.build_summary()
# ------------- Loss Function ------------------
# F_loss=model.classify_loss_src+model.dregular_loss+model.G_loss \
# + model.class_wise_adaptation_loss \
# + alpha2*(model.sintra_loss+model.tintra_loss-0.1*model.sinter_loss-0.1*model.tinter_loss)
# F_loss=model.classify_loss_src+model.dregular_loss+alpha1*model.G_loss \
# + alpha1*model.class_wise_adaptation_loss \
# + alpha2*(model.sintra_loss+model.tintra_loss-0.1*model.sinter_loss-0.1*model.tinter_loss)
F_loss=model.classify_loss_src+model.gregular_loss+alpha1*model.G_loss \
+ alpha1*model.class_wise_adaptation_loss \
+ alpha2*(model.sintra_loss+model.tintra_loss)
# Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
if args.no_update_mean_var == True:
update_ops = None
else:
print(toMagenta('updating mean and var in batchnorm'))
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
opt_1_1 = tf.train.MomentumOptimizer(lr1, args.momentum)
grads_1_1 = tf.gradients(F_loss, fine_tune_var_weights)
train_op_1_1 = opt_1_1.apply_gradients(zip(grads_1_1, fine_tune_var_weights))
opt_1_2 = tf.train.MomentumOptimizer(2*lr1, args.momentum)
grads_1_2 = tf.gradients(F_loss, fine_tune_var_bias)
train_op_1_2 = opt_1_2.apply_gradients(zip(grads_1_2, fine_tune_var_bias))
opt_2_1 = tf.train.MomentumOptimizer(lr2, args.momentum)
grads_2_1 = tf.gradients(F_loss, retrain_var_weights)
train_op_2_1 = opt_2_1.apply_gradients(zip(grads_2_1, retrain_var_weights))
opt_2_2 = tf.train.MomentumOptimizer(2*lr2, args.momentum)
grads_2_2 = tf.gradients(F_loss, retrain_var_bias)
train_op_2_2 = opt_2_2.apply_gradients(zip(grads_2_2, retrain_var_bias))
opt_3_1 = tf.train.MomentumOptimizer(lr2,args.momentum)
grads_3_1 = tf.gradients(model.dregular_loss+model.D_loss, domain_var_weights)
train_op_3_1 = opt_3_1.apply_gradients(zip(grads_3_1, domain_var_weights))
opt_3_2 = tf.train.MomentumOptimizer(2*lr2,args.momentum)
grads_3_2 = tf.gradients(model.dregular_loss+model.D_loss, domain_var_bias)
train_op_3_2 = opt_3_2.apply_gradients(zip(grads_3_2, domain_var_bias))
train_op = tf.group(train_op_1_1,train_op_1_2,train_op_2_1,train_op_2_2,train_op_3_1,train_op_3_2)
# Set up tf session and initialize variables.
#
config = tf.ConfigProto()#Chong
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.4
sess = tf.Session(config=config)
init_local=tf.local_variables_initializer()
init = tf.global_variables_initializer()
# construct summary
summary_.append(tf.summary.scalar(
'train/lr1', lr1))
summary_.append(tf.summary.scalar(
'train/lr2', lr2))
summary_.append(tf.summary.scalar(
'train/alpha1', alpha1))
summary_.append(tf.summary.scalar(
'train/alpha2', alpha2))
summary_merged=tf.summary.merge(summary_)
FinalSummary = tf.summary.FileWriter(args.snapshot_dir,sess.graph)
# init
sess.run([init_local,init])
# Saver for storing checkpoints of the model.
var=tf.global_variables()
skip_var=['fc8','fc9']
saver = tf.train.Saver(var_list=var, max_to_keep=5)
ckpt = tf.train.get_checkpoint_state(args.resume_from)
if ckpt and ckpt.model_checkpoint_path and args.resume:
loader = tf.train.Saver(var_list=var)
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
elif not args.not_load_pretrained:
print(toRed('Restore from pre-trained model...' + args.restore_from))
model.load_initial_weights(sess, args.restore_from, skip_var) #Chong:0531
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
acc2_history=0
for step in range(start_steps,num_steps):
start_time = time.time()
feed_dict = {step_ph: step, keep_prob: 0.5}
summary, total_loss, _= sess.run([summary_merged,F_loss,train_op], feed_dict=feed_dict)
FinalSummary.add_summary(summary, step)
duration = time.time() - start_time
remain_time=duration*(num_steps-step)/3600
print('\r',toCyan('{:s}:{:d}-{:d}-{:d} total loss = {:.3f},({:.3f} sec/step, ERT: {:.3f})'.format(args.model_name+'-'+args.domain,step%steps_per_epoch, step//steps_per_epoch,args.num_epochs,total_loss, duration,remain_time)),end='')
if step % args.test_every == 0:
acc1,acc2=0,0
for jj in range(val_num_steps):
feed_dict = {keep_prob: 1}
src_acc,tar_acc= sess.run([model.src_acc,model.tar_acc], feed_dict=feed_dict)
acc1+=np.sum(src_acc)
acc2+=np.sum(tar_acc)
acc1=acc1/(val_num_steps*args.batch_size)
acc2=acc2/(val_num_steps*args.batch_size)
# pdb.set_trace()
test_summary = tf.Summary()
test_summary.value.add(tag='test/source_accuracy',simple_value= acc1)
test_summary.value.add(tag='test/target_accuracy',simple_value= acc2)
FinalSummary.add_summary(test_summary, step)
if acc2>acc2_history:
save(saver, sess, args.snapshot_dir, step)
acc2_history=acc2
coord.request_stop()
coord.join(threads)
sess.close()
def main():
# Create model and start training
h, w = INPUT_SIZE
X = tf.placeholder(tf.float32, shape=[None, h, w, 3], name='X')
Y = tf.placeholder(tf.uint8, shape=[None, h, w, 1], name='Y')
is_training = tf.placeholder(tf.bool, name='is_training')
net = DeepLabResNetModel(X, is_training, NUM_CLASSES, ATROUS_BLOCKS)
raw_output = net.output
# Trainable Variables
# restore_vars = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
all_trainable = [
v for v in tf.trainable_variables()
if 'beta' not in v.name and 'gamma' not in v.name
]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable if 'fc' not in v.name]
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name]
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name]
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, NUM_CLASSES])
label_proc = prepare_labels(Y,
tf.stack(raw_output.get_shape()[1:3]),
num_classes=NUM_CLASSES,
one_hot=False)
raw_gt = tf.reshape(label_proc, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, NUM_CLASSES - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise Softmax Loss
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
labels=gt)
l2_losses = [
WEIGHT_DECAY * tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'weights' in v.name
]
reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
variable_summaries(reduced_loss, name='loss')
variable_summaries(loss, name='loss_origin')
# Processed predictions: for visualization
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(X)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Define loss and optimization parameters
base_lr = tf.constant(LEARNING_RATE, tf.float64)
global_step = tf.Variable(0, trainable=False, name='global_step')
increment_step = tf.assign(global_step, global_step + 1)
learning_rate = tf.scalar_mul(base_lr,
tf.pow((1 - global_step / NUM_STEPS), POWER))
learning_rate = tf.maximum(learning_rate, 8e-7)
opt_conv = tf.train.MomentumOptimizer(learning_rate, MOMENTUM)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 5.0, MOMENTUM)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 10.0, MOMENTUM)
grads = tf.gradients(reduced_loss,
conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) +
len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = tf.group(increment_step, train_op_conv, train_op_fc_w,
train_op_fc_b)
# initial_learning_rate = 1e-2
# learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step, 300, 0.96)
# adam = tf.train.AdamOptimizer(learning_rate).minimize(reduced_loss, global_step=global_step)
# Image Summary
images_summary = tf.py_func(inv_preprocess, [X, SAVE_NUM_IMAGES, IMG_MEAN],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[pred, SAVE_NUM_IMAGES, NUM_CLASSES], tf.uint8)
labels_summary = tf.py_func(decode_labels,
[Y, SAVE_NUM_IMAGES, NUM_CLASSES], tf.uint8)
image_summaries = [images_summary, preds_summary, labels_summary]
image_summary = tf.summary.image('images',
tf.concat(axis=2, values=image_summaries),
max_outputs=SAVE_NUM_IMAGES)
# Variable Summary
variable_summaries(fc_w_trainable, 'fc_w')
variable_summaries(fc_b_trainable, 'fc_b')
variable_summaries(learning_rate, 'learning_rate')
# variable_summaries(net.weights, 'aconv_w')
# variable_summaries(net.biases, 'aconv_b')
total_summary = tf.summary.merge_all()
tb_train_dir = os.path.join(SNAPSHOT_DIR, 'train')
tb_val_dir = os.path.join(SNAPSHOT_DIR, 'verify')
summary_writer = tf.summary.FileWriter(tb_train_dir,
graph=tf.get_default_graph())
verify_writer = tf.summary.FileWriter(tb_val_dir)
train_data, val_data = read_data()
train_data_count = len(train_data)
batch_size = BATCH_SIZE
print('batch size: %s, total step: %s, train data num: %s' %
(batch_size, NUM_STEPS, train_data_count))
# Set up session
with tf.Session() as sess:
tf.global_variables_initializer().run()
saver = tf.train.Saver(max_to_keep=3)
if SNAPSHOT_DIR is not None and os.path.exists(SNAPSHOT_DIR):
loader = tf.train.Saver()
load_model(loader, sess, SNAPSHOT_DIR)
# export pb
if False:
import export_pb
export_pb.save_graph_with_weight(
sess, ['ExpandDims'],
'/data/pb/traffic_line_deeplab_resnet_cut05_271_loss_0_0_93.pb'
)
start_index = 0
for step in range(NUM_STEPS):
# start_time = time.time()
if start_index + batch_size > train_data_count:
start_index = 0
random.shuffle(train_data)
_train_data = train_data[start_index:start_index + batch_size]
start_index += batch_size
after_processing_data = []
start_time = time.time()
for x_img, y_data in _train_data:
# tmp_img = augmentation(x_img)
# x_data = np.asarray(tmp_img).astype('float32') / 255.0
after_processing_data.append((x_img, y_data))
vec = [d[0] for d in after_processing_data]
vec2 = [d[1] for d in after_processing_data]
# data_process_time = time.time() - start_time
if step % SAVE_SUMMARY_EVERY == 0:
feed = [
reduced_loss, pred, total_summary, global_step, train_op
]
loss_value, preds, summary, total_steps, _ = \
sess.run(feed, feed_dict={X: vec, Y: vec2, is_training: True})
summary_writer.add_summary(summary, total_steps)
else:
feed = [reduced_loss, global_step, train_op]
loss_value, total_steps, _ = sess.run(feed,
feed_dict={
X: vec,
Y: vec2,
is_training: True
})
if step % SAVE_MODEL_EVERY == 0:
save_model(saver, sess, SNAPSHOT_DIR, global_step)
duration = time.time() - start_time
results = 'global step: {:d}, step: {:d} \t loss = {:.3f}, ({:.3f} secs)'\
.format(total_steps, step, loss_value, duration)
if step % WRITE_EVERY == 0:
with open(WRITE_FILE, 'a') as f:
f.write(results + '\n')
print(results)
if step % VAL_EVERY == 0:
random.shuffle(val_data)
vec = [d[0] for d in val_data[:batch_size]]
vec2 = [d[1] for d in val_data[:batch_size]]
feed = [reduced_loss, pred, total_summary, global_step]
loss_value, preds, summary, total_steps = \
sess.run(feed, feed_dict={X: vec, Y: vec2, is_training: False})
verify_writer.add_summary(summary, total_steps)
print('[Verify]', step, loss_value)
def __init__(self,
num_classes,
vocab_size,
shape_domain_size,
char_domain_size,
char_size,
embedding_size,
shape_size,
nonlinearity,
viterbi,
hidden_dim,
char_embeddings,
embeddings=None):
self.num_classes = num_classes
self.shape_domain_size = shape_domain_size
self.char_domain_size = char_domain_size
self.char_size = char_size
self.embedding_size = embedding_size
self.shape_size = shape_size
self.hidden_dim = hidden_dim
self.nonlinearity = nonlinearity
self.char_embeddings = char_embeddings
self.viterbi = viterbi
# word embedding input
self.input_x1 = tf.placeholder(tf.int64, [None, None], name="input_x1")
# shape embedding input
self.input_x2 = tf.placeholder(tf.int64, [None, None], name="input_x2")
# labels
self.input_y = tf.placeholder(tf.int64, [None, None], name="input_y")
# padding mask
self.input_mask = tf.placeholder(tf.float32, [None, None],
name="input_mask")
self.batch_size = tf.placeholder(tf.int32, None, name="batch_size")
self.max_seq_len = tf.placeholder(tf.int32, None, name="max_seq_len")
# sequence lengths
self.sequence_lengths = tf.placeholder(tf.int32, [None, None],
name="sequence_lengths")
# dropout and l2 penalties
self.middle_dropout_keep_prob = tf.placeholder_with_default(
1.0, [], name="middle_dropout_keep_prob")
self.hidden_dropout_keep_prob = tf.placeholder_with_default(
1.0, [], name="hidden_dropout_keep_prob")
self.input_dropout_keep_prob = tf.placeholder_with_default(
1.0, [], name="input_dropout_keep_prob")
self.word_dropout_keep_prob = tf.placeholder_with_default(
1.0, [], name="word_dropout_keep_prob")
self.l2_penalty = tf.placeholder_with_default(0.0, [],
name="l2_penalty")
self.projection = tf.placeholder_with_default(False, [],
name="projection")
self.drop_penalty = tf.placeholder_with_default(0.0, [],
name="drop_penalty")
# Keeping track of l2 regularization loss (optional)
self.l2_loss = tf.constant(0.0)
# set the pad token to a constant 0 vector
self.word_zero_pad = tf.constant(0.0,
dtype=tf.float32,
shape=[1, embedding_size])
self.shape_zero_pad = tf.constant(0.0,
dtype=tf.float32,
shape=[1, shape_size])
self.char_zero_pad = tf.constant(0.0,
dtype=tf.float32,
shape=[1, char_size])
self.use_characters = char_size != 0
self.use_shape = shape_size != 0
if self.viterbi:
self.transition_params = tf.get_variable(
"transitions", [num_classes, num_classes])
# Embedding layer
# with tf.device('/cpu:0'), tf.name_scope("embedding"):
word_embeddings_shape = (vocab_size - 1, embedding_size)
self.w_e = tf_utils.initialize_embeddings(word_embeddings_shape,
name="w_e",
pretrained=embeddings)
nonzero_elements = tf.not_equal(self.sequence_lengths,
tf.zeros_like(self.sequence_lengths))
count_nonzero_per_row = tf.reduce_sum(tf.to_int32(nonzero_elements),
axis=1)
self.flat_sequence_lengths = tf.add(
tf.reduce_sum(self.sequence_lengths, 1),
tf.scalar_mul(2, count_nonzero_per_row))
self.unflat_scores, self.hidden_layer = self.forward(
self.input_x1,
self.input_x2,
self.max_seq_len,
self.hidden_dropout_keep_prob,
self.input_dropout_keep_prob,
self.middle_dropout_keep_prob,
reuse=False)
# Calculate mean cross-entropy loss
with tf.name_scope("loss"):
labels = tf.cast(self.input_y, 'int32')
if viterbi:
log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(
self.unflat_scores,
labels,
self.flat_sequence_lengths,
transition_params=self.transition_params)
# self.transition_params = transition_params
self.loss = tf.reduce_mean(-log_likelihood)
else:
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.unflat_scores, labels=labels)
masked_losses = tf.multiply(losses, self.input_mask)
self.loss = tf.div(tf.reduce_sum(masked_losses),
tf.reduce_sum(self.input_mask))
self.loss += self.l2_penalty * self.l2_loss
self.unflat_no_dropout_scores, _ = self.forward(
self.input_x1, self.input_x2, self.max_seq_len, 1.0, 1.0, 1.0)
drop_loss = tf.nn.l2_loss(
tf.subtract(self.unflat_scores, self.unflat_no_dropout_scores))
self.loss += self.drop_penalty * drop_loss
# Accuracy
with tf.name_scope("predictions"):
if viterbi:
self.predictions = self.unflat_scores
else:
self.predictions = tf.argmax(self.unflat_scores, 2)
def train():
learning_rate = args.lr
model_path = args.model
total_epoch = args.epoch
teacher = nin()
student = lenet()
if args.noisy == True:
drop_scale = 1 / args.Nratio
noisy_mask = tf.nn.dropout(tf.constant(
np.float32(np.ones((batch_size, 1))) / drop_scale),
keep_prob=args.Nratio) #(batchsize,1)
gaussian = tf.random_normal(shape=[batch_size, 1],
mean=0.0,
stddev=args.Nsigma)
noisy = tf.mul(noisy_mask, gaussian)
#noisy_add = tf.add(tf.constant(np.float32(np.ones((batch_size,1)))), noisy)
teacher = tf.mul(teacher,
tf.tile(noisy, tf.constant([1, 10]))) #(batchsize,10)
#teacher = tf.add(teacher, tf.tile(noisy,tf.constant([1,10])))
print(bcolors.G + "prepare for training, noisy mode" + bcolors.END)
tf_loss = tf.nn.l2_loss(teacher - student) / batch_size
elif args.KD == True: # correct Hinton method at 2017.1.3
print(bcolors.G + "prepare for training, knowledge distilling mode" +
bcolors.END)
one_hot = tf.one_hot(y, n_classes, 1.0, 0.0)
#one_hot = tf.cast(one_hot_int, tf.float32)
teacher_tau = tf.scalar_mul(1.0 / args.tau, teacher)
student_tau = tf.scalar_mul(1.0 / args.tau, student)
objective1 = tf.nn.sigmoid_cross_entropy_with_logits(
student_tau, one_hot)
objective2 = tf.scalar_mul(0.5, tf.square(student_tau - teacher_tau))
tf_loss = (args.lamda * tf.reduce_sum(objective1) +
(1 - args.lamda) * tf.reduce_sum(objective2)) / batch_size
else:
print(bcolors.G + "prepare for training, NIPS2014 mode" + bcolors.END)
tf_loss = tf.nn.l2_loss(teacher - student) / batch_size
optimizer1 = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(tf_loss)
optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate /
10).minimize(tf_loss)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
sess = tf.InteractiveSession(config=tf.ConfigProto(
gpu_options=gpu_options, allow_soft_placement=True))
tf.initialize_all_variables().run()
with tf.device('/cpu:0'):
saver = tf.train.Saver(max_to_keep=100)
#saver.restore(sess, os.path.join(model_path,'model-99')
data, label = read_cifar10('train')
index = np.array(range(len(data))) # index randomly ordered
mean = cal_mean()
begin = time.time()
iterations = len(data) / batch_size
decay_step = int(total_epoch * 0.8)
cnt = 0
dropout_rate = args.dropout
print(bcolors.G + "number of iterations (per epoch) =" +
str(len(data) / batch_size) + bcolors.END)
for i in range(total_epoch):
np.random.shuffle(index)
cost_sum = 0
for j in range(iterations):
batch_x = np.float32(
data[index[j * batch_size:(j + 1) * batch_size]]) - mean
batch_y = np.squeeze(
np.float32(label[index[j * batch_size:(j + 1) * batch_size]]))
if cnt / decay_step == 0:
lr = learning_rate
_, cost = sess.run([optimizer1, tf_loss],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1 - dropout_rate
})
elif cnt / decay_step == 1:
lr = learning_rate / 10
_, cost = sess.run([optimizer2, tf_loss],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1 - dropout_rate
})
cost_sum += cost
#pdb.set_trace()
#if (j % int(iterations*0.25) == 0):
# print ("epoch %d-iter %d, cost = %f , avg-cost = %f"%(i, j, cost, cost/n_classes))
# sys.stdout.flush()
cnt += 1
avg_time = time.time() - begin
print(
"epoch %d - avg. %f seconds in each epoch, lr = %.0e, cost = %f , avg-cost-per-logits = %f"
% (i, avg_time / cnt, lr, cost_sum,
cost_sum / iterations / n_classes))
if np.mod(i + 1, 10) == 0:
print("Epoch ", i + 1, " is done. Saving the model ...")
with tf.device('/cpu:0'):
if not os.path.exists(model_path):
os.makedirs(model_path)
saver.save(sess,
os.path.join(model_path, 'model'),
global_step=i)
sys.stdout.flush()
def loss(self, predictions_dict, groundtruth_dict):
"""Define loss functions given the prediction and the groundtruth dictionary of tensors
Args:
predictions_dict: a dictionary holding predicted tensors
groundtruth_dict: a dictionary holding groundtruth tensors for computing the loss
Returns:
loss_dict: a dictionary mapping strings (loss names) to scalar tensors representing
loss values.
"""
loss_config = self.model_config.losses
batch_size = self.model_config.batch_size
batch_reduce_factor = 2
if self.model_config.input_queue_type in ['source_only', 'target_only']:
batch_reduce_factor = 1
loss_dict = {
'triplet_loss': None,
'domain_loss': None,
'classify_loss': None,
'reconstruction_loss': None,
'wasserstein_loss': None,
'kl_loss': None
}
if loss_config.reconstruction_loss.use_reconstruction_loss:
reconstruction_loss = losses.reconstruction_loss(inputs=predictions_dict[PREPROCESSED_INPUTS_KEY],
outputs=predictions_dict[PRED_RECONSTRUCTION_KEY],
scale=float(loss_config.reconstruction_loss.reconstruction_loss_weight))
loss_dict['reconstruction_loss'] = reconstruction_loss
if loss_config.triplet_loss.use_triplet_loss:
triplet_loss = losses.triplet_loss(labels=tf.argmax(groundtruth_dict[mnist_dataset_builder.CLASS_LABEL_KEY], axis=1),
embeddings=predictions_dict[EMBEDDING_FEATURES_KEY],
margin=float(loss_config.triplet_loss.triplet_loss_margin),
scale=float(loss_config.triplet_loss.triplet_loss_weight),
batch_reduce_factor=batch_reduce_factor)
loss_dict['triplet_loss'] = triplet_loss
if loss_config.domain_loss.use_domain_loss:
domain_loss, domain_accuracy, _ = losses.softmax_cross_entropy_with_logits_v2(
logits=predictions_dict[PRED_DOMAIN_KEY],
labels=groundtruth_dict[
mnist_dataset_builder.DOMAIN_LABEL_KEY],
scale=float(
loss_config.domain_loss.domain_loss_weight), loss_name='Loss/domain_loss')
loss_dict['domain_loss'] = domain_loss
# TODO(): Add classification_accuracy to the tensorboard summary
if loss_config.source_classification_loss.use_source_classification_loss:
source_classify_loss, classify_source_accuracy, classify_target_accuracy = losses.softmax_cross_entropy_with_logits_v2(
logits=predictions_dict[PRED_CLASSES_KEY], labels=groundtruth_dict[mnist_dataset_builder.CLASS_LABEL_KEY],
scale=float(loss_config.source_classification_loss.source_classification_loss_weight),
batch_reduce_factor=batch_reduce_factor, loss_name='Loss/source_classify_loss')
loss_dict['classify_loss'] = source_classify_loss
# TODO(): Recheck the output of the wasserstein loss, adjust the batch size
if loss_config.wasserstein_loss.use_wasserstein_loss:
wasser_loss = -tf.contrib.gan.losses.wargs.wasserstein_discriminator_loss(
discriminator_real_outputs=predictions_dict[PRED_DOMAIN_KEY][:batch_size // batch_reduce_factor],
discriminator_gen_outputs=predictions_dict[PRED_DOMAIN_KEY][batch_size // batch_reduce_factor:])
wasser_loss = tf.scalar_mul(loss_config.wasserstein_loss.wasserstein_loss_weight, wasser_loss)
loss_dict['wasserstein_loss'] = wasser_loss
if loss_config.kl_divergence_loss.use_kl_divergence_loss:
kl_loss = losses.kl_div(predictions_dict[EMBEDDING_FEATURES_KEY][:batch_size // batch_reduce_factor],
predictions_dict[EMBEDDING_FEATURES_KEY][batch_size // batch_reduce_factor:])
kl_loss = tf.scalar_mul(loss_config.use_kl_divergence_loss.kl_divergence_loss_weight, kl_loss)
loss_dict['kl_loss'] = kl_loss
# loss = triplet_loss + domain_loss + reconstruction_loss + classify_loss
#
# loss_dict = {
# 'triplet_loss': triplet_loss,
# 'domain_loss': domain_loss,
# 'classify_loss': source_classify_loss,
# 'reconstruction_loss': reconstruction_loss,
# 'wasserstein_loss': wasser_loss,
# 'kl_loss': kl_loss
# }
return loss_dict
def build_network(d):
# Define hyperparameters
d = d
learning_rate = 2e-5
l2norm_scaling = 1e-10
global_norm_gradient_clipping_ratio = 0.65
# Define a placeholder for the answers to the decision problems
route_exists = tf.placeholder( tf.float32, shape = (None,), name = 'route_exists' )
# Define a placeholder for the cost of each route
route_costs = tf.placeholder( tf.float32, shape=(None,1), name='route_costs')
# Define a placeholder for the edges mask
edges_mask = tf.placeholder( tf.float32, shape = (None,), name = 'edges_mask' )
# Define placeholders for the list of number of vertices and edges per instance
n_vertices = tf.placeholder( tf.int32, shape = (None,), name = 'n_vertices')
n_edges = tf.placeholder( tf.int32, shape = (None,), name = 'edges')
#
EV_matrix = tf.placeholder( tf.float32, shape = (None,None), name = "EV" )
edge_weight = tf.placeholder( tf.float32, shape = (None,1), name = "edge_weight" )
target_cost = tf.placeholder( tf.float32, shape = (None,1), name = "target_cost" )
time_steps = tf.placeholder( tf.int32, shape = (), name = "time_steps" )
initial_embedding_mlp = Mlp(
layer_sizes = [ d for _ in range(3) ],
activations = [ tf.nn.relu for _ in range(3) ],
output_size = d,
name = 'E_init_MLP',
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
edge_initial_embeddings = initial_embedding_mlp(
tf.concat(
[ edge_weight, target_cost ],
axis = 1
)
)
vertex_initial_embeddings = tf.get_variable(
initializer = tf.random_normal( (1,d) ),
dtype = tf.float32, name='V_init'
)
total_n = tf.shape( EV_matrix )[1]
tiled_and_normalized_vertex_initial_embeddings = tf.tile(
tf.div(
vertex_initial_embeddings,
tf.sqrt( tf.cast( total_n, tf.float32 ) )
),
[ total_n, 1 ]
)
# Define GNN dictionary
GNN = {}
# Define Graph neural network
gnn = GraphNN(
{
# V is the set of vertex embeddings
'V': d,
# E is the set of edge embeddings
'E': d
},
{
# M is a E×V adjacency matrix connecting each edge to the vertices it is connected to
'EV': ('E','V')
},
{
# V_msg_E is a MLP which computes messages from vertex embeddings to edge embeddings
'V_msg_E': ('V','E'),
# E_msg_V is a MLP which computes messages from edge embeddings to vertex embeddings
'E_msg_V': ('E','V')
},
{
# V(t+1) ← Vu( EVᵀ × E_msg_V(E(t)) )
'V': [
{
'mat': 'EV',
'msg': 'E_msg_V',
'transpose?': True,
'var': 'E'
}
],
# E(t+1) ← Eu( EV × V_msg_E(V(t)), W, C )
'E': [
{
'mat': 'EV',
'msg': 'V_msg_E',
'var': 'V'
}
]
},
name='TSP'
)
# Populate GNN dictionary
GNN['gnn'] = gnn
GNN['route_exists'] = route_exists
GNN['route_costs'] = route_costs
GNN['edges_mask'] = edges_mask
GNN['n_vertices'] = n_vertices
GNN['n_edges'] = n_edges
GNN["EV"] = EV_matrix
GNN["W"] = edge_weight
GNN["C"] = target_cost
GNN["time_steps"] = time_steps
# Define E_vote, which will compute one logit for each edge
E_vote_MLP = Mlp(
layer_sizes = [ d for _ in range(3) ],
activations = [ tf.nn.relu for _ in range(3) ],
output_size = 1,
name = 'E_vote',
name_internal_layers = True,
kernel_initializer = tf.contrib.layers.xavier_initializer(),
bias_initializer = tf.zeros_initializer()
)
vote_bias = tf.get_variable(initializer=tf.zeros_initializer(), shape=(), dtype=tf.float32, name='vote_bias')
# Get the last embeddings
last_states = gnn(
{ "EV": EV_matrix },
{ "V": tiled_and_normalized_vertex_initial_embeddings, "E": edge_initial_embeddings },
time_steps = time_steps
)
GNN["last_states"] = last_states
E_n = last_states['E'].h
# Compute a vote for each embedding
#E_vote = tf.reshape(E_vote_MLP(tf.concat([E_n,route_costs], axis=1)), [-1])
E_vote = tf.reshape(E_vote_MLP(E_n), [-1])
E_prob = tf.sigmoid(E_vote)
# Compute the number of problems in the batch
num_problems = tf.shape(n_vertices)[0]
# n_edges_acc[i] is the number of edges in the batch up to the i-th instance
n_edges_acc = tf.map_fn(lambda i: tf.reduce_sum(tf.gather(n_edges, tf.range(0,i))), tf.range(0,num_problems))
# Compute decision predictions (one per problem)
_, pred_logits = tf.while_loop(
lambda i, predictions: tf.less(i, num_problems),
lambda i, predictions:
(
(i+1),
predictions.write(
i,
#tf.reduce_mean(tf.gather(E_vote, tf.range(n_edges_acc[i], n_edges_acc[i] + n_edges[i])))
tf.reduce_mean( E_vote[n_edges_acc[i]:n_edges_acc[i]+n_edges[i]] )
)
),
[0, tf.TensorArray(size=num_problems, dtype=tf.float32)]
)
pred_logits = pred_logits.stack() + vote_bias
GNN['predictions'] = tf.sigmoid(pred_logits)
# Count the number of edges that appear in the solution
pos_edges_n = tf.reduce_sum(edges_mask)
# Count the number of edges that do not appear in the solution
neg_edges_n = tf.reduce_sum(tf.subtract(tf.ones_like(edges_mask), edges_mask))
# Define edges loss
GNN['loss_edges'] = tf.losses.sigmoid_cross_entropy(
multi_class_labels = edges_mask,
logits = E_vote,
weights = tf.add(
tf.scalar_mul(
tf.divide(tf.add(pos_edges_n,neg_edges_n),pos_edges_n),
edges_mask),
tf.scalar_mul(
tf.divide(tf.add(pos_edges_n,neg_edges_n),neg_edges_n),
tf.subtract(tf.ones_like(edges_mask), edges_mask)
)
)
)
# Define decision loss
GNN['loss_decision'] = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=route_exists, logits=pred_logits))
# Compute true positives, false positives, true negatives, false negatives
GNN['true_pos'] = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['false_pos'] = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['true_neg'] = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
GNN['false_neg'] = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
# Define edges accuracy
GNN['acc_edges'] = tf.reduce_mean(tf.cast(tf.equal(edges_mask, tf.round(E_prob)), tf.float32))
# Define decision accuracy
GNN['acc_decision'] = tf.reduce_mean(tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32))
# Define optimizer
optimizer = tf.train.AdamOptimizer(name='Adam', learning_rate=learning_rate)
# Compute cost relative to L2 normalization
vars_cost = tf.add_n([ tf.nn.l2_loss(var) for var in tf.trainable_variables() ])
# Define gradients and train step
for loss_type in ['edges','decision']:
grads, _ = tf.clip_by_global_norm(tf.gradients(GNN['loss_' + loss_type] + tf.multiply(vars_cost, l2norm_scaling),tf.trainable_variables()),global_norm_gradient_clipping_ratio)
GNN['train_step_' + loss_type] = optimizer.apply_gradients(zip(grads, tf.trainable_variables()))
#end
# Return GNN dictionary
return GNN
def _static_subsample(self, indicator, batch_size, labels):
"""Returns subsampled minibatch.
Args:
indicator: boolean tensor of shape [N] whose True entries can be sampled.
N should be a complie time constant.
batch_size: desired batch size. This scalar cannot be None.
labels: boolean tensor of shape [N] denoting positive(=True) and negative
(=False) examples. N should be a complie time constant.
Returns:
sampled_idx_indicator: boolean tensor of shape [N], True for entries which
are sampled. It ensures the length of output of the subsample is always
batch_size, even when number of examples set to True in indicator is
less than batch_size.
Raises:
ValueError: if labels and indicator are not 1D boolean tensors.
"""
# Check if indicator and labels have a static size.
if not indicator.shape.is_fully_defined():
raise ValueError(
'indicator must be static in shape when is_static is'
'True')
if not labels.shape.is_fully_defined():
raise ValueError('labels must be static in shape when is_static is'
'True')
if not isinstance(batch_size, int):
raise ValueError(
'batch_size has to be an integer when is_static is'
'True.')
input_length = tf.shape(indicator)[0]
# Set the number of examples set True in indicator to be at least
# batch_size.
num_true_sampled = tf.reduce_sum(tf.cast(indicator, tf.float32))
additional_false_sample = tf.less_equal(
tf.cumsum(tf.cast(tf.logical_not(indicator), tf.float32)),
batch_size - num_true_sampled)
indicator = tf.logical_or(indicator, additional_false_sample)
# Shuffle indicator and label. Need to store the permutation to restore the
# order post sampling.
permutation = tf.random_shuffle(tf.range(input_length))
indicator = ops.matmul_gather_on_zeroth_axis(
tf.cast(indicator, tf.float32), permutation)
labels = ops.matmul_gather_on_zeroth_axis(tf.cast(labels, tf.float32),
permutation)
# index (starting from 1) when indicator is True, 0 when False
indicator_idx = tf.where(tf.cast(indicator, tf.bool),
tf.range(1, input_length + 1),
tf.zeros(input_length, tf.int32))
# Replace -1 for negative, +1 for positive labels
signed_label = tf.where(
tf.cast(labels, tf.bool), tf.ones(input_length, tf.int32),
tf.scalar_mul(-1, tf.ones(input_length, tf.int32)))
# negative of index for negative label, positive index for positive label,
# 0 when indicator is False.
signed_indicator_idx = tf.multiply(indicator_idx, signed_label)
sorted_signed_indicator_idx = tf.nn.top_k(signed_indicator_idx,
input_length,
sorted=True).values
[num_positive_samples, num_negative_samples
] = self._get_num_pos_neg_samples(sorted_signed_indicator_idx,
batch_size)
sampled_idx = self._get_values_from_start_and_end(
sorted_signed_indicator_idx, num_positive_samples,
num_negative_samples, batch_size)
# Shift the indices to start from 0 and remove any samples that are set as
# False.
sampled_idx = tf.abs(sampled_idx) - tf.ones(batch_size, tf.int32)
sampled_idx = tf.multiply(
tf.cast(tf.greater_equal(sampled_idx, tf.constant(0)), tf.int32),
sampled_idx)
sampled_idx_indicator = tf.cast(
tf.reduce_sum(tf.one_hot(sampled_idx, depth=input_length), axis=0),
tf.bool)
# project back the order based on stored permutations
reprojections = tf.one_hot(permutation,
depth=input_length,
dtype=tf.float32)
return tf.cast(
tf.tensordot(tf.cast(sampled_idx_indicator, tf.float32),
reprojections,
axes=[0, 0]), tf.bool)
def add_prediction_op(self):
"""
Adds the unrolled RNN:
h_0 = 0
for t in 1 to T:
o_t, h_t = cell(x_t, h_{t-1})
o_drop_t = Dropout(o_t, dropout_rate)
y_t = o_drop_t U + b_2
Returns:
pred: tf.Tensor of shape (batch_size, max_length, non_terminal_vocab)
"""
x = self.add_embedding()
dropout_rate = self.dropout_placeholder
preds = [] # Predicted output at each timestep should go here!
hidden = []
cell = LSTMCell(Config.n_token_features * Config.embed_size,
Config.hidden_size)
# Define U and b2 as variables.
# Initialize state as vector of zeros.
xinit = tf.contrib.layers.xavier_initializer(dtype=tf.float64)
if not self.config.terminal_pred:
output_size = self.config.non_terminal_vocab
else:
output_size = self.config.terminal_vocab
U = tf.get_variable('U',
shape=[self.config.hidden_size, output_size],
initializer=xinit,
dtype=tf.float64)
b2 = tf.get_variable('b2',
shape=[output_size],
initializer=tf.constant_initializer(0.0),
dtype=tf.float64)
c_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size],
dtype=tf.float64)
h_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size],
dtype=tf.float64)
state_tuple = (c_t, h_t)
scope = "LSTM_terminal" if self.config.terminal_pred else "LSTM_non_terminal"
with tf.variable_scope(scope):
for time_step in range(self.max_length):
if time_step > 0:
tf.get_variable_scope().reuse_variables()
o_t, h_t = cell(x[:, time_step, :], state_tuple)
o_drop_t = tf.nn.dropout(o_t, dropout_rate)
preds.append(tf.matmul(o_drop_t, U) + b2)
hidden.append(h_t[1])
if not (self.config.cell
== "lstmAend") and not (self.config.cell == "lstm"):
W_a = tf.get_variable('W_a',
shape=[
self.config.hidden_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
W_o = tf.get_variable(
'W_o',
shape=[2 * self.config.hidden_size, output_size],
dtype=tf.float64,
initializer=xinit)
W_s = tf.get_variable('W_s',
shape=[output_size, output_size],
dtype=tf.float64,
initializer=xinit)
b_o = tf.get_variable(
'b_o',
shape=[output_size],
dtype=tf.float64,
initializer=tf.constant_initializer(0.0))
b_s = tf.get_variable(
'b_s',
shape=[output_size],
dtype=tf.float64,
initializer=tf.constant_initializer(0.0))
hidden_stack = tf.stack(hidden, 1)
ht = tf.reshape(
tf.matmul(h_t[1], W_a),
(tf.shape(x)[0], -1, self.config.hidden_size))
weights = tf.reduce_sum(
ht * hidden_stack, axis=2) * tf.slice(
self.attn_mask_placeholder, [0, 0],
[-1, time_step + 1])
weights = tf.nn.softmax(weights)
context = tf.reduce_sum(tf.reshape(
weights,
(tf.shape(weights)[0], tf.shape(weights)[1], -1)) *
hidden_stack,
axis=1)
context_hidden_sum = tf.add(context, h_t[1])
if (self.config.cell == "lstmAcont"):
hidden = hidden[:-1] + [context]
if (self.config.cell == "lstmAsum_fn"):
W_alpha = tf.get_variable('W_alpha',
shape=[
self.config.hidden_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
W_beta = tf.get_variable('W_beta',
shape=[
self.config.hidden_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
U_alpha = tf.get_variable('U_alpha',
shape=[
self.config.embed_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
U_beta = tf.get_variable('U_beta',
shape=[
self.config.embed_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
alpha = tf.sigmoid(
tf.matmul(context, W_alpha) +
tf.matmul(x[:, time_step, :], U_alpha))
beta = tf.sigmoid(
tf.matmul(h_t[1], W_beta) +
tf.matmul(x[:, time_step, :], U_beta))
straightSum = alpha * context + beta * h_t[1]
hidden = hidden[:-1] + [straightSum]
if (self.config.cell == "lstmAsum"):
alpha = tf.get_variable(
'alpha',
shape=(),
dtype=tf.float64,
initializer=tf.random_uniform_initializer(
-1.0, 2.0))
beta = tf.get_variable(
'beta',
shape=(),
dtype=tf.float64,
initializer=tf.random_uniform_initializer(
-1.0, 2.0))
straightSum = tf.add(tf.scalar_mul(alpha, context),
tf.scalar_mul(beta, h_t[1]))
hidden = hidden[:-1] + [straightSum]
if (self.config.cell == "lstmAwsum_fn"):
W_ph = tf.get_variable('W_ph',
shape=[
self.config.hidden_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
W_pc = tf.get_variable('W_pc',
shape=[
self.config.hidden_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
W_px = tf.get_variable('W_px',
shape=[
self.config.embed_size,
self.config.hidden_size
],
dtype=tf.float64,
initializer=xinit)
hTerm = tf.matmul(h_t[1], W_ph)
cTerm = tf.matmul(context, W_pc)
xTerm = tf.matmul(x[:, time_step, :], W_px)
p_arr = tf.sigmoid(hTerm + cTerm + xTerm)
weightedSum = (p_arr * context) + (
(1 - p_arr) * h_t[1])
hidden = hidden[:-1] + [weightedSum]
preds = tf.stack(preds, 1)
hidden = tf.stack(hidden, 1)
final_preds = tf.boolean_mask(preds, self.mask_placeholder)
final_hidden = tf.boolean_mask(hidden, self.mask_placeholder)
if not (self.config.cell == "lstm"):
W_a = tf.get_variable(
'W_a',
shape=[self.config.hidden_size, self.config.hidden_size],
dtype=tf.float64,
initializer=xinit)
W_o = tf.get_variable(
'W_o',
shape=[2 * self.config.hidden_size, output_size],
dtype=tf.float64,
initializer=xinit)
W_s = tf.get_variable('W_s',
shape=[output_size, output_size],
dtype=tf.float64,
initializer=xinit)
b_o = tf.get_variable('b_o',
shape=[output_size],
dtype=tf.float64,
initializer=tf.constant_initializer(0.0))
b_s = tf.get_variable('b_s',
shape=[output_size],
dtype=tf.float64,
initializer=tf.constant_initializer(0.0))
ht = tf.reshape(tf.matmul(final_hidden, W_a),
(tf.shape(x)[0], -1, self.config.hidden_size))
weights = tf.reduce_sum(ht * hidden,
axis=2) * self.attn_mask_placeholder
weights = tf.nn.softmax(weights)
context = tf.reduce_sum(
tf.reshape(weights,
(tf.shape(weights)[0], tf.shape(weights)[1], -1)) *
hidden,
axis=1)
final_preds = tf.tanh(
tf.matmul(tf.concat(1, [context, final_hidden]), W_o) + b_o)
final_preds = tf.matmul(final_preds, W_s) + b_s
if self.config.terminal_pred:
nt = tf.nn.embedding_lookup(
self.embeddings, self.next_non_terminal_input_placeholder)
nt = tf.reshape(
nt,
[-1, self.config.n_token_features * self.config.embed_size])
U_nt = tf.get_variable(
'U_nt',
shape=[self.config.hidden_size, output_size],
initializer=xinit,
dtype=tf.float64)
b_t = tf.get_variable('b_t',
shape=[output_size],
initializer=tf.constant_initializer(0.0),
dtype=tf.float64)
final_preds = final_preds + tf.matmul(nt, U_nt) + b_t
return final_preds
def fit(self, mu, x, y_desired):
y_pred = self(x)
e = y_desired - y_pred
x = tf.constant(np.array([1.0] + x, dtype=np.float32))
self.w.assign_add(tf.scalar_mul(2 * mu * e, x))
def main():
"""Create the model and start the training."""
args = get_arguments()
"""
Get configurations here. We pass some arguments from command line to init configurations, for training hyperparameters,
you can set them in TrainConfig Class.
Note: we set filter scale to 1 for pruned model, 2 for non-pruned model. The filters numbers of non-pruned
model is two times larger than prunde model, e.g., [h, w, 64] <-> [h, w, 32].
"""
cfg = TrainConfig(dataset=args.dataset,
is_training=True,
random_scale=args.random_scale,
random_mirror=args.random_mirror,
filter_scale=args.filter_scale)
if args.num_classes is not None:
cfg.param["num_classes"] = args.num_classes
if args.data_dir is not None:
cfg.param["data_dir"] = args.data_dir
if args.val_list is not None:
cfg.param["eval_list"] = args.val_list
if args.train_list is not None:
cfg.param["train_list"] = args.train_list
if args.ignore_label is not None:
cfg.param["ignore_label"] = args.ignore_label
if args.eval_size is not None:
cfg.param["eval_size"] = [
int(x.strip()) for x in args.eval_size.split("x")[::-1]
]
if args.training_size is not None:
cfg.TRAINING_SIZE = [
int(x.strip()) for x in args.training_size.split("x")[::-1]
]
if args.batch_size is not None:
cfg.BATCH_SIZE = args.batch_size
if args.learning_rate is not None:
cfg.LEARNING_RATE = args.learning_rate
if args.restore_from is not None:
cfg.model_weight = args.restore_from
if args.snapshot_dir is not None:
cfg.SNAPSHOT_DIR = args.snapshot_dir
if args.restore_from == "scratch":
from tqdm import tqdm
import cv2
import joblib as joblib
if not args.img_mean:
print(
"Calculating img mean for custom dataset. To prevent this, specify it with --img-mean next time"
)
image_files, annotation_files = read_labeled_image_list(
cfg.param["data_dir"], cfg.param["train_list"])
means = joblib.Parallel(n_jobs=6)(
joblib.delayed(calc_mean)(image_file, cv2)
for image_file in tqdm(image_files, desc="calc img mean"))
cfg.IMG_MEAN = np.mean(means, axis=0).tolist()
else:
cfg.IMG_MEAN = [float(x.strip()) for x in args.img_mean.split(",")]
cfg.display()
# Setup training network and training samples
train_reader = ImageReader(cfg=cfg, mode='train')
train_net = ICNet_BN(image_reader=train_reader, cfg=cfg, mode='train')
loss_sub4, loss_sub24, loss_sub124, reduced_loss = create_losses(
train_net, train_net.labels, cfg)
# Setup validation network and validation samples
with tf.variable_scope('', reuse=True):
val_reader = ImageReader(cfg, mode='eval')
val_net = ICNet_BN(image_reader=val_reader, cfg=cfg, mode='train')
val_loss_sub4, val_loss_sub24, val_loss_sub124, val_reduced_loss = create_losses(
val_net, val_net.labels, cfg)
# Using Poly learning rate policy
base_lr = tf.constant(cfg.LEARNING_RATE)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(
base_lr, tf.pow((1 - step_ph / cfg.TRAINING_STEPS), cfg.POWER))
# Set restore variable
restore_var = tf.global_variables()
all_trainable = [
v for v in tf.trainable_variables()
if ('beta' not in v.name and 'gamma' not in v.name)
or args.train_beta_gamma
]
# Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
if args.update_mean_var == False:
update_ops = None
else:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
opt_conv = tf.train.MomentumOptimizer(learning_rate, cfg.MOMENTUM)
grads = tf.gradients(reduced_loss, all_trainable)
train_op = opt_conv.apply_gradients(zip(grads, all_trainable))
# Create session & restore weights (Here we only need to use train_net to create session since we reuse it)
train_net.create_session()
if args.initializer:
train_net.set_initializer(initializer_algorithm=args.initializer)
train_net.initialize_variables()
if not args.restore_from or args.restore_from != "scratch":
train_net.restore(cfg.model_weight, restore_var)
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=20)
total_parameters = 0
for variable in tf.trainable_variables():
# shape is an array of tf.Dimension
shape = variable.get_shape()
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
total_parameters += variable_parameters
print("Total trainable parameters: " + str(total_parameters))
# Iterate over training steps.
val_loss_value = 10.0
min_val_loss = float("inf")
stagnation = 0
max_non_decreasing_val_loss = int(
np.ceil(args.early_stopping_patience * len(train_reader.image_list) /
(cfg.BATCH_SIZE * cfg.EVAL_EVERY)))
print(
"Maximum times that val loss can stagnate before early stopping is applied: "
+ str(max_non_decreasing_val_loss))
for step in range(cfg.TRAINING_STEPS):
start_time = time.time()
feed_dict = {step_ph: step}
if step % cfg.EVAL_EVERY == 0:
loss_value, loss1, loss2, loss3, val_loss_value, _ = train_net.sess.run(
[
reduced_loss, loss_sub4, loss_sub24, loss_sub124,
val_reduced_loss, train_op
],
feed_dict=feed_dict)
if val_loss_value < min_val_loss:
print("New best val loss {:.3f}. Saving weights...".format(
val_loss_value))
train_net.save(
saver,
cfg.SNAPSHOT_DIR,
step,
model_name="val{:.3f}model.ckpt".format(val_loss_value))
min_val_loss = val_loss_value
stagnation = 0
else:
stagnation += 1
else:
loss_value, loss1, loss2, loss3, _ = train_net.sess.run(
[reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op],
feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f}, val_loss: {:.3f} ({:.3f} sec/step)'.\
format(step, loss_value, loss1, loss2, loss3, val_loss_value, duration))
if stagnation > max_non_decreasing_val_loss:
print("Early stopping")
break
def floatToFixPoint(tensor_): tensor_ = tf.clip_by_value(tensor_,-32.0,32.0) tensor_ = tf.scalar_mul(67108864.0,tensor_) tensor_ = tf.round(tensor_) tensor_ = tf.scalar_mul(1/67108864.0,tensor_) return (tensor_)
def mnist_fid_score(self, epoch, fid_train_images_names):
def data_reader_faces(filename):
with tf.device('/CPU'):
print(tf.cast(filename[0], dtype=tf.string))
image_string = tf.io.read_file(
tf.cast(filename[0], dtype=tf.string))
# Don't use tf.image.decode_image, or the output shape will be undefined
image = tf.image.decode_jpeg(image_string, channels=3)
image.set_shape([218, 178, 3])
image = tf.image.crop_to_bounding_box(image, 38, 18, 140, 140)
image = tf.image.resize(image, [80, 80])
# This will convert to float values in [0, 1]
image = tf.subtract(image, 127.5)
image = tf.divide(image, 127.5)
return image
self.load_fid_model()
fid_num_samples = 1000
fid_batch_size = tf.constant(100, dtype='int64')
num_parallel_calls = 4
random_points = tf.keras.backend.random_uniform(
[min(fid_train_images_names.shape[0], fid_num_samples)],
minval=0,
maxval=int(fid_train_images_names.shape[0]),
dtype='int32',
seed=None)
fid_train_images_names = fid_train_images_names[random_points]
fid_image_dataset = tf.data.Dataset.from_tensor_slices(
fid_train_images_names)
fid_image_dataset = fid_image_dataset.map(
data_reader_faces, num_parallel_calls=int(num_parallel_calls))
fid_image_dataset = fid_image_dataset.batch(fid_batch_size)
with tf.device(config.DEVICE):
count = 0
fid_sum = 0
for image_batch in fid_image_dataset:
noise = tf.random.normal([fid_batch_size, self.noise_dim],
self.noise_mean, self.noise_stddev)
preds = self.generator(noise, training=False)
preds = tf.image.resize(preds, [80, 80])
preds = tf.scalar_mul(2., preds)
preds = tf.subtract(preds, 1.0)
preds = preds.numpy()
act1 = self.fid_model.predict(image_batch)
act2 = self.fid_model.predict(preds)
try:
act1 = np.concatenate((act1, act1), axis=0)
act2 = np.concatenate((act2, act2), axis=0)
fid_score = self.calculate_fid(act1, act2)
fid_sum += fid_score
count += 1
except:
act1 = act1
act2 = act2
avg_fid_score = fid_sum / count / fid_batch_size
self.fid_scores.append(
str(epoch) + ',' + str(np.array(avg_fid_score)))
print("epoch: %d fid: %f" % (epoch, avg_fid_score))
def train_setup(self):
tf.set_random_seed(self.conf.random_seed)
# Create queue coordinator.
self.coord = tf.train.Coordinator()
# Input size
h, w = (self.conf.input_height, self.conf.input_width)
input_size = (h, w)
# Devices
gpu_list = get_available_gpus()
zip_encoder, zip_decoder_b, zip_decoder_w, zip_crf = [], [], [], []
previous_crf_names = []
restore_vars = []
self.loaders = []
self.im_list = []
for i in range(len(gpu_list)):
with tf.device(gpu_list[i]):
# Load reader
with tf.name_scope("create_inputs"):
reader = ImageReader(self.conf.data_dir,
self.conf.data_list, input_size,
self.conf.random_scale,
self.conf.random_mirror,
self.conf.ignore_label, IMG_MEAN,
self.coord)
self.image_batch, self.label_batch = reader.dequeue(
self.conf.batch_size)
self.im_list.append(self.image_batch)
image_batch_075 = tf.image.resize_images(
self.image_batch,
[int(h * 0.75), int(w * 0.75)])
image_batch_05 = tf.image.resize_images(
self.image_batch,
[int(h * 0.5), int(w * 0.5)])
# Create network
with tf.variable_scope('', reuse=False):
net = Deeplab_v2(self.image_batch,
self.conf.num_classes,
True,
rescale075=False,
rescale05=False,
crf_type=self.conf.crf_type)
with tf.variable_scope('', reuse=True):
net075 = Deeplab_v2(image_batch_075,
self.conf.num_classes,
True,
rescale075=True,
rescale05=False,
crf_type=self.conf.crf_type)
with tf.variable_scope('', reuse=True):
net05 = Deeplab_v2(image_batch_05,
self.conf.num_classes,
True,
rescale075=False,
rescale05=True,
crf_type=self.conf.crf_type)
# Variables that load from pre-trained model.
restore_var = [
v for v in tf.global_variables()
if ('fc' not in v.name and 'crfrnn' not in v.name)
]
restore_vars.append(restore_var)
# Trainable Variables
all_trainable = tf.trainable_variables()
# Fine-tune part
for name in previous_crf_names:
for v in all_trainable:
if v.name == name:
all_trainable.remove(v)
crf_trainable = [
v for v in all_trainable
if ('crfrnn' in v.name and v.name not in previous_crf_names
)
]
previous_crf_names.extend(v.name for v in crf_trainable)
encoder_trainable = [
v for v in all_trainable
if 'fc' not in v.name and 'crfrnn' not in v.name
] # lr * 1.0
# Remove encoder_trainable from all_trainable
#all_trainable = [v for v in all_trainable if v not in encoder_trainable]
# Decoder part
decoder_trainable = [
v for v in all_trainable
if 'fc' in v.name and 'crfrnn' not in v.name
]
decoder_w_trainable = [
v for v in decoder_trainable
if ('weights' in v.name or 'gamma' in v.name)
and 'crfrnn' not in v.name
] # lr * 10.0
decoder_b_trainable = [
v for v in decoder_trainable
if ('biases' in v.name or 'beta' in v.name)
and 'crfrnn' not in v.name
] # lr * 20.0
# Check
assert (len(all_trainable) == len(decoder_trainable) +
len(crf_trainable)) + len(encoder_trainable)
assert (len(decoder_trainable) == len(decoder_w_trainable) +
len(decoder_b_trainable))
# Network raw output
raw_output100 = net.outputs
raw_output075 = net075.outputs
raw_output05 = net05.outputs
raw_output = tf.reduce_max(tf.stack([
raw_output100,
tf.image.resize_images(raw_output075,
tf.shape(raw_output100)[1:3, ]),
tf.image.resize_images(raw_output05,
tf.shape(raw_output100)[1:3, ])
]),
axis=0)
# Ground Truth: ignoring all labels greater or equal than n_classes
label_proc = prepare_label(self.label_batch,
tf.stack(
raw_output.get_shape()[1:3]),
num_classes=self.conf.num_classes,
one_hot=True) # [batch_size, h, w]
label_proc075 = prepare_label(
self.label_batch,
tf.stack(raw_output075.get_shape()[1:3]),
num_classes=self.conf.num_classes,
one_hot=True)
label_proc05 = prepare_label(
self.label_batch,
tf.stack(raw_output05.get_shape()[1:3]),
num_classes=self.conf.num_classes,
one_hot=True)
raw_gt = tf.reshape(label_proc, [
-1,
])
raw_gt075 = tf.reshape(label_proc075, [
-1,
])
raw_gt05 = tf.reshape(label_proc05, [
-1,
])
indices = tf.squeeze(
tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)),
1)
indices075 = tf.squeeze(
tf.where(
tf.less_equal(raw_gt075, self.conf.num_classes - 1)),
1)
indices05 = tf.squeeze(
tf.where(tf.less_equal(raw_gt05,
self.conf.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32)
gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32)
raw_prediction = tf.reshape(raw_output,
[-1, self.conf.num_classes])
raw_prediction100 = tf.reshape(raw_output100,
[-1, self.conf.num_classes])
raw_prediction075 = tf.reshape(raw_output075,
[-1, self.conf.num_classes])
raw_prediction05 = tf.reshape(raw_output05,
[-1, self.conf.num_classes])
prediction = tf.gather(raw_prediction, indices)
prediction100 = tf.gather(raw_prediction100, indices)
prediction075 = tf.gather(raw_prediction075, indices075)
prediction05 = tf.gather(raw_prediction05, indices05)
# Pixel-wise softmax_cross_entropy loss
#loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
loss = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=raw_prediction,
labels=tf.reshape(label_proc[0],
(h * w, self.conf.num_classes)))
'''
coefficients = [0.01460247, 1.25147725, 2.88479363, 1.20348121, 1.65261654, 1.67514772,
0.62338799, 0.7729363, 0.42038501, 0.98557268, 1.31867536, 0.85313332,
0.67227604, 1.21317965, 1. , 0.24263748, 1.80877607, 1.3082213,
0.79664027, 0.72543945, 1.27823374]
'''
#loss = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc[0], (h*w, self.conf.num_classes)), logits=raw_prediction)
#loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction100, labels=gt)
loss100 = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=raw_prediction100,
labels=tf.reshape(label_proc[0],
(h * w, self.conf.num_classes)))
#loss100 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc[0], (h*w, self.conf.num_classes)), logits=raw_prediction100)
#loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction075, labels=gt075)
loss075 = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=raw_prediction075,
labels=tf.reshape(label_proc075[0],
(int(h * 0.75) * int(w * 0.75),
self.conf.num_classes)))
#loss075 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc075[0], (int(h * 0.75) * int(w * 0.75), self.conf.num_classes)), logits=raw_prediction075)
#loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction05, labels=gt05)
loss05 = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=raw_prediction05,
labels=tf.reshape(
label_proc05[0],
(int(h * 0.5) * int(w * 0.5), self.conf.num_classes)))
#loss05 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc05[0], (int(h * 0.5) * int(w * 0.5), self.conf.num_classes)), logits=raw_prediction05)
# L2 regularization
l2_losses = [
self.conf.weight_decay * tf.nn.l2_loss(v)
for v in all_trainable if 'weights' in v.name
]
# Loss function
self.reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean(
loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(
loss05) + tf.add_n(l2_losses)
# Define optimizers
# 'poly' learning rate
base_lr = tf.constant(self.conf.learning_rate)
self.curr_step = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(
base_lr,
tf.pow((1 - self.curr_step / self.conf.num_steps),
self.conf.power))
# We have several optimizers here in order to handle the different lr_mult
# which is a kind of parameters in Caffe. This controls the actual lr for each
# layer.
opt_encoder = tf.train.MomentumOptimizer(
learning_rate, self.conf.momentum)
opt_decoder_w = tf.train.MomentumOptimizer(
learning_rate * 10.0, self.conf.momentum)
opt_decoder_b = tf.train.MomentumOptimizer(
learning_rate * 20.0, self.conf.momentum)
opt_crf = tf.train.MomentumOptimizer(learning_rate,
self.conf.momentum)
# Gradient accumulation
# Define a variable to accumulate gradients.
accum_grads = [
tf.Variable(tf.zeros_like(v.initialized_value()),
trainable=False) for v in encoder_trainable +
decoder_w_trainable + decoder_b_trainable + crf_trainable
]
# Define an operation to clear the accumulated gradients for next batch.
self.zero_op = [
v.assign(tf.zeros_like(v)) for v in accum_grads
]
# To make sure each layer gets updated by different lr's, we do not use 'minimize' here.
# Instead, we separate the steps compute_grads+update_params.
# Compute grads
grads = tf.gradients(
self.reduced_loss, encoder_trainable +
decoder_w_trainable + decoder_b_trainable + crf_trainable)
# Accumulate and normalise the gradients.
self.accum_grads_op = [
accum_grads[i].assign_add(grad /
self.conf.grad_update_every)
for i, grad in enumerate(grads)
]
grads_encoder = accum_grads[:len(encoder_trainable)]
grads_decoder_w = accum_grads[len(encoder_trainable
):len(encoder_trainable) +
len(decoder_w_trainable)]
grads_decoder_b = accum_grads[(
len(encoder_trainable) +
len(decoder_w_trainable)):(len(encoder_trainable) +
len(decoder_w_trainable) +
len(decoder_b_trainable))]
grads_crf = accum_grads[
len(encoder_trainable) + len(decoder_w_trainable) +
len(decoder_b_trainable
):] # assuming crf gradients are appended to the end
zip_encoder.append(list(zip(grads_encoder, encoder_trainable)))
zip_decoder_b.append(
list(zip(grads_decoder_b, decoder_b_trainable)))
zip_decoder_w.append(
list(zip(grads_decoder_w, decoder_w_trainable)))
zip_crf.append(list(zip(grads_crf, crf_trainable)))
avg_grads_encoder = average_gradients(zip_encoder)
avg_grads_decoder_w = average_gradients(zip_decoder_w)
avg_grads_decoder_b = average_gradients(zip_decoder_b)
avg_grads_crf = average_gradients(zip_crf)
for i in range(len(gpu_list)):
with tf.device(gpu_list[i]):
# Update params
train_op_conv = opt_encoder.apply_gradients(avg_grads_encoder)
train_op_fc_w = opt_decoder_w.apply_gradients(
avg_grads_decoder_w)
train_op_fc_b = opt_decoder_b.apply_gradients(
avg_grads_decoder_b)
train_op_crf = opt_crf.apply_gradients(avg_grads_crf)
# Finally, get the train_op!
update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS
) # for collecting moving_mean and moving_variance
with tf.control_dependencies(update_ops):
self.train_op = tf.group(train_op_fc_w, train_op_fc_b,
train_op_crf) # train_op_conv
# Saver for storing checkpoints of the model
self.saver = tf.train.Saver(var_list=tf.global_variables(),
max_to_keep=0)
# Loader for loading the pre-trained model
for i in range(len(gpu_list)):
with tf.device(gpu_list[i]):
self.loaders.append(tf.train.Saver(var_list=restore_vars[i]))
#self.loaders.append(tf.train.Saver(var_list=tf.global_variables()))
# Training summary
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, input_size)
raw_output_up = tf.argmax(raw_output_up, axis=3)
self.pred = tf.expand_dims(raw_output_up, axis=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[self.image_batch, 1, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(
decode_labels, [self.label_batch, 1, self.conf.num_classes],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[self.pred, 1, self.conf.num_classes],
tf.uint8)
self.total_summary = tf.summary.image(
'images',
tf.concat(axis=2,
values=[images_summary, labels_summary, preds_summary]),
max_outputs=1) # Concatenate row-wise.
if not os.path.exists(self.conf.logdir):
os.makedirs(self.conf.logdir)
self.summary_writer = tf.summary.FileWriter(
self.conf.logdir, graph=tf.get_default_graph())
def preprocess_for_train(image, classes, boxes, resolution, speed_mode=False):
if speed_mode:
pass
else:
image_ori = image
classes_ori = classes
boxes_ori = boxes
# randomly sample patches
if RANDOM_SUB_SAMPLE:
image_shape = tf.shape(image)
x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1)
x1 = tf.expand_dims(x1, -1)
x2 = tf.expand_dims(x2, -1)
y1 = tf.expand_dims(y1, -1)
y2 = tf.expand_dims(y2, -1)
boxes = tf.concat([y1, x1, y2, x2], axis=1)
bbox_begin, bbox_size, distort_bbox = tf.image.sample_distorted_bounding_box(
image_shape,
bounding_boxes=tf.expand_dims(boxes, 0),
min_object_covered=BBOX_CROP_OVERLAP,
aspect_ratio_range=(0.5, 2.0),
area_range=(0.3, 1.0),
max_attempts=200,
use_image_if_no_bounding_boxes=True)
distort_bbox = distort_bbox[0, 0]
image = tf.slice(image, bbox_begin, bbox_size)
boxes = bboxes_resize(distort_bbox, boxes)
classes, boxes = bboxes_filter_overlap(classes,
boxes,
threshold=BBOX_CROP_OVERLAP,
assign_negative=False)
y1, x1, y2, x2 = tf.unstack(boxes, 4, axis=1)
image_shape = tf.shape(image)
x1 = tf.clip_by_value(x1, 0.0, 1.0)
x2 = tf.clip_by_value(x2, 0.0, 1.0)
y1 = tf.clip_by_value(y1, 0.0, 1.0)
y2 = tf.clip_by_value(y2, 0.0, 1.0)
x1 = tf.expand_dims(x1, -1)
x2 = tf.expand_dims(x2, -1)
y1 = tf.expand_dims(y1, -1)
y2 = tf.expand_dims(y2, -1)
boxes = tf.concat([x1, y1, x2, y2], axis=1)
if RANDOM_ZOOM_OUT:
image, boxes = random_zoom_out(image, boxes)
if RANDOM_HFLIP:
# Randomly flip the image horizontally.
image, boxes = random_flip_left_right(image, boxes)
if RANDOM_COLOR:
# Color distortion
image = tf.div(image, 255.0)
image = apply_with_random_selector(
image,
lambda x, ordering: distort_color(x, ordering, fast_mode=False
),
num_cases=4)
image = tf.scalar_mul(255.0, image)
# Rollback to the full image if there is no boxes
no_box_cond = tf.equal(tf.size(boxes), 0)
image = tf.cond(no_box_cond, lambda: image_ori, lambda: image)
boxes = tf.cond(no_box_cond, lambda: boxes_ori, lambda: boxes)
classes = tf.cond(no_box_cond, lambda: classes_ori, lambda: classes)
# Scaling to canonical size without perspective preserved
image, scale, translation = aspect_preserving_resize(
image, resolution, depth=3, resize_mode="bilinear")
new_image = tf.image.resize_image_with_crop_or_pad(
image, resolution, resolution)
scale_x = tf.to_float(tf.shape(image)[1]) / tf.to_float(resolution)
scale_y = tf.to_float(tf.shape(image)[0]) / tf.to_float(resolution)
shift_x = tf.math.maximum(
0.0,
tf.to_float(resolution - tf.shape(image)[1]) /
float(resolution * 2))
shift_y = tf.math.maximum(
0.0,
tf.to_float(resolution - tf.shape(image)[0]) /
float(resolution * 2))
x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1)
x1 = tf.scalar_mul(scale_x, x1)
y1 = tf.scalar_mul(scale_y, y1)
x2 = tf.scalar_mul(scale_x, x2)
y2 = tf.scalar_mul(scale_y, y2)
boxes = tf.concat([
tf.expand_dims(x1, -1),
tf.expand_dims(y1, -1),
tf.expand_dims(x2, -1),
tf.expand_dims(y2, -1)
],
axis=1)
boxes = boxes + [shift_x, shift_y, shift_x, shift_y]
image = new_image
# Scaling to canonical size without preserving perspective
# image, scale, translation = bilinear_resize(image, resolution, depth=3, resize_mode="bilinear")
# mean subtraction
means = [_R_MEAN, _G_MEAN, _B_MEAN]
channels = tf.split(axis=2, num_or_size_splits=3, value=image)
for i in range(3):
channels[i] -= means[i]
# image = tf.concat(axis=2, values=channels)
# caffe swaps color channels
image = tf.concat(axis=2,
values=[channels[2], channels[1], channels[0]])
return image, classes, boxes, scale, translation
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
coord = tf.train.Coordinator()
with tf.name_scope("create_inputs"):
reader = ImageReader(
DATA_DIR,
DATA_LIST_PATH,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale)
sub4_out = net.layers['sub4_out']
sub24_out = net.layers['sub24_out']
sub124_out = net.layers['conv6_cls']
restore_var = tf.global_variables()
# restore_var = [v for v in tf.global_variables() if 'conv6_cls' not in v.name]
all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma]
loss_sub4 = create_loss(sub4_out, label_batch, args.num_classes, args.ignore_label)
loss_sub24 = create_loss(sub24_out, label_batch, args.num_classes, args.ignore_label)
loss_sub124 = create_loss(sub124_out, label_batch, args.num_classes, args.ignore_label)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
reduced_loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses)
# Using Poly learning rate policy
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
# Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
if args.update_mean_var == False:
update_ops = None
else:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
grads = tf.gradients(reduced_loss, all_trainable)
train_op = opt_conv.apply_gradients(zip(grads, all_trainable))
# Set up tf session and initialize variables.
# 先初始化所有变量
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=5)
ckpt = tf.train.get_checkpoint_state(args.snapshot_dir)
if ckpt and ckpt.model_checkpoint_path:
loader = tf.train.Saver(var_list=restore_var)
load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('Restore from pre-trained model...')
net.load(args.restore_from, sess)
# args.restore_from:./model/icnet_cityscapes_trainval_90k_bnnomerge.npy
# net_data = np.load(open(args.restore_from,"rb"),encoding="latin1").item()
# loader = tf.train.Saver(var_list=exclude_restore_var)
# loader.restore(sess, net_data)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
if step % args.save_pred_every == 0:
loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration))
coord.request_stop()
coord.join(threads)
for i in range(args.train_batch):
fake_labels[i, tmp_fake_labels[i]] = 1
loss = categorical_crossentropy(net.y_, net.y)
top1 = categorical_accuracy(net.y_, net.y)
top5 = top_k_categorical_accuracy(net.y_, net.y, 5)
base_lr = 0.02
step = tf.Variable(0, trainable=False, name="Step")
learning_rate = tf.train.exponential_decay(base_lr, step, 1, 0.999964)
weight_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "W:0"]
bias_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "b:0"]
optimizer1 = tf.train.MomentumOptimizer(learning_rate, 0.9)
optimizer2 = tf.train.MomentumOptimizer(tf.scalar_mul(2.0, learning_rate), 0.9)
grads = optimizer1.compute_gradients(loss, var_list=weight_list+bias_list)
w_grads = grads[:len(weight_list)]
b_grads = grads[len(weight_list):]
train1 = optimizer1.apply_gradients(w_grads, global_step=step)
train2 = optimizer2.apply_gradients(b_grads, global_step=step)
train_step = tf.group(train1, train2)
init = tf.global_variables_initializer()
sess = tf.Session()
K.set_session(sess)
sess.run(init)
start = time.time()
dataset = tf.data.Dataset.from_tensor_slices((imNames, imLabels, imSplit))
dataset = dataset.map(parseFunction,
num_parallel_calls=options.numParallelLoaders)
dataset = dataset.shuffle(buffer_size=numItemsInDataset, seed=0)
dataset = dataset.batch(options.batchSize)
iterator = dataset.make_initializable_iterator()
with tf.name_scope('Model'):
# Data placeholders
inputBatchImages, inputBatchImageNames, inputBatchImageLabels, inputBatchImageSplit = iterator.get_next(
)
print("Data shape: %s" % str(inputBatchImages.get_shape()))
if (options.modelName == "IncResV2") or (options.modelName == "NASNet"):
scaledInputBatchImages = tf.scalar_mul((1.0 / 255), inputBatchImages)
scaledInputBatchImages = tf.subtract(scaledInputBatchImages, 0.5)
scaledInputBatchImages = tf.multiply(scaledInputBatchImages, 2.0)
# Create model
if options.modelName == "IncResV2":
arg_scope = inception_resnet_v2.inception_resnet_v2_arg_scope()
with slim.arg_scope(arg_scope):
logits, aux_logits, endPoints = inception_resnet_v2.inception_resnet_v2(
scaledInputBatchImages, is_training=False)
# Get the lower layer and upper layer activations
lowerLayerActivations = endPoints["?"]
upperLayerActivations = endPoints["?"]
# Create list of vars to restore before train op