def create_cost(c, w, b, nb, length, weight_spacing = 1.0, weight_bending = 1.0, gamma = 1.0, kappa = 2.0): #tangents t = create_tangent(c); tn = create_normalize_tangent(t); nl = create_normal(tn); nr = tf.scalar_mul(-1.0, nl); l,r = create_left_right(c,w,nl); cost_left = create_cost_soft_min_aligned_distance(l, b, nl, nb, k = kappa, gamma = gamma); cost_right= create_cost_soft_min_aligned_distance(r, b, nr, nb, k = kappa, gamma = gamma); cost = tf.add(cost_left, cost_right); #spacing and bending if weight_spacing != 0: cost_spacing = tf.scalar_mul(weight_spacing, create_cost_spacing(t, length)); cost = tf.add(cost, cost_spacing); else: cost_spacing = tf.constant(0); if weight_bending != 0: cost_bending = tf.scalar_mul(weight_bending, create_cost_bending(tn)); cost = tf.add(cost, cost_bending); else: cost_bending = tf.constant(0); return (cost, cost_left, cost_right, cost_spacing, cost_bending, nl, l, r);
def clip_norm(g, c, n): if c <= 0: # if clipnorm == 0 no need to add ops to the graph return g # tf require using a special op to multiply IndexedSliced by scalar if K.backend() == 'tensorflow': condition = n >= c then_expression = tf.scalar_mul(c / n, g) else_expression = g # saving the shape to avoid converting sparse tensor to dense if isinstance(then_expression, tf.Tensor): g_shape = copy.copy(then_expression.get_shape()) elif isinstance(then_expression, tf.IndexedSlices): g_shape = copy.copy(then_expression.dense_shape) if condition.dtype != tf.bool: condition = tf.cast(condition, 'bool') g = tf.cond(condition, lambda: then_expression, lambda: else_expression) if isinstance(then_expression, tf.Tensor): g.set_shape(g_shape) elif isinstance(then_expression, tf.IndexedSlices): g._dense_shape = g_shape else: g = K.switch(K.greater_equal(n, c), g * c / n, g) return g
def predict_slim(sample_images, print_func=print): """ Code modified from here: [https://github.com/tensorflow/models/issues/429] """ # Setup preprocessing input_tensor = tf.placeholder(tf.float32, shape=(None, 299, 299, 3), name='input_image') scaled_input_tensor = tf.scalar_mul((1.0 / 255), input_tensor) scaled_input_tensor = tf.subtract(scaled_input_tensor, 0.5) scaled_input_tensor = tf.multiply(scaled_input_tensor, 2.0) # Setup session sess = tf.Session() arg_scope = slim_irv2.inception_resnet_v2_arg_scope() with slim.arg_scope(arg_scope): _, end_points = slim_irv2.inception_resnet_v2(scaled_input_tensor, is_training=False) # Load the model print_func("Loading TF-slim checkpoint...") saver = tf.train.Saver() saver.restore(sess, SLIM_CKPT) # Make prediction predict_values = [] for image in sample_images: im = Image.open(image).resize((299, 299)) arr = np.expand_dims(np.array(im), axis=0) y_pred = sess.run([end_points['Predictions']], feed_dict={input_tensor: arr}) y_pred = y_pred[0].ravel() y_pred = y_pred[1:] / y_pred[1:].sum() # remove background class and renormalize print_func("{} class={} prob={}".format(image, np.argmax(y_pred), np.max(y_pred))) predict_values.append(y_pred) return predict_values
def thresholding(inputs): # find the mean for each example in the batch mean_output = tf.reduce_mean(inputs, axis=1) # scale each mean based on a factor threshold_scalar = tf.Variable(utils.threshold_scalar, tf.float32) scaled_mean = tf.scalar_mul(threshold_scalar, mean_output) scaled_mean = tf.reshape(scaled_mean, [utils.batch_size]) # setup matrix for min_thresh_for_max = tf.fill([utils.batch_size], 0.05) max_thresh_for_min = tf.fill([utils.batch_size], 0.15) #0.4 thresholds = tf.maximum(min_thresh_for_max, scaled_mean) thresholds = tf.minimum(max_thresh_for_min, thresholds) # zero values under the thresholds using bitmask thresholds = tf.reshape(thresholds, [128, 1, 1]) threshold_mask = tf.cast(tf.greater(inputs, thresholds), tf.float32) thresholded_input = tf.multiply(inputs, threshold_mask) # peak picking # select beats by x[i-1] < x[i] > x[i+1] (local maximum) x_minus_1 = tf.cast(tf.greater(thresholded_input, tf.manip.roll(thresholded_input, shift=-1, axis=1)), tf.float32) x_plus_1 = tf.cast(tf.greater(thresholded_input, tf.manip.roll(thresholded_input, shift=1, axis=1)), tf.float32) output = tf.multiply(x_minus_1, x_plus_1) return output
def focal_loss(onehot_labels, cls_preds, alpha=0.25, gamma=2.0, name=None, scope=None): """Compute softmax focal loss between logits and onehot labels logits and onehot_labels must have same shape [batchsize, num_classes] and the same data type (float16, 32, 64) Args: onehot_labels: Each row labels[i] must be a valid probability distribution cls_preds: Unscaled log probabilities alpha: The hyperparameter for adjusting biased samples, default is 0.25 gamma: The hyperparameter for penalizing the easy labeled samples name: A name for the operation (optional) Returns: A 1-D tensor of length batch_size of same type as logits with softmax focal loss """ with tf.name_scope(scope, 'focal_loss', [cls_preds, onehot_labels]) as sc: logits = tf.convert_to_tensor(cls_preds) onehot_labels = tf.convert_to_tensor(onehot_labels) precise_logits = tf.cast(logits, tf.float32) if ( logits.dtype == tf.float16) else logits onehot_labels = tf.cast(onehot_labels, precise_logits.dtype) predictions = tf.nn.sigmoid(logits) predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1. - predictions) # add small value to avoid 0 epsilon = 1e-8 alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32)) alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1 - alpha_t) losses = tf.reduce_sum( -alpha_t * tf.pow(1. - predictions_pt, gamma) * onehot_labels * tf.log(predictions_pt + epsilon), name=name, axis=1) return losses
def build_graph(self,test_decoder_logits): print('starting building graph [sentiment-discriminator]') with tf.variable_scope("sentiment") as scope: self.inputs = tf.slice(test_decoder_logits,[0,0,0],[self.batch_size,self.max_length,self.vocab_size]) # variable weights = { 'w2v' : tf.get_variable(initializer = tf.random_uniform_initializer(-0.1, 0.1, dtype=tf.float32),shape = [self.vocab_size, self.embedding_dim], name='w2v'), 'out_1' : tf.get_variable(initializer = tf.random_normal_initializer(), shape = [self.unit_size*2, 1], name='w_out_1'), } biases = { 'out_1' : tf.get_variable(initializer = tf.random_normal_initializer(), shape=[1], name='b_out_1'), } # structure def BiRNN(x): x = tf.unstack(x, self.max_length, 1) lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(self.unit_size, forget_bias=1.0) lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(self.unit_size,forget_bias=1.0) outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32 ) return outputs[-1] self.inputs_softmax = tf.nn.softmax(tf.scalar_mul(tf.constant(5.0, shape=[]),self.inputs)) y_list=[] for i in range(self.inputs.get_shape().as_list()[0]): y = tf.matmul(self.inputs_softmax[i], weights['w2v']) y = tf.reshape(y, [1, self.max_length, self.embedding_dim]) y_list.append(y) embbed_layer = tf.concat(y_list,0) layer_1 = BiRNN(embbed_layer) pred = tf.matmul(layer_1, weights['out_1']) + biases['out_1'] # get score self.score = tf.sigmoid(pred)
def test_decoder_loop(prev,i): factor = tf.constant(5,shape=(),dtype=tf.float32) prev = tf.scalar_mul(factor,tf.add(tf.matmul(prev,weight_output),bias_output)) prev_index = tf.nn.softmax(prev) pred_prev = tf.matmul(prev_index,word_embedding_matrix) next_input = pred_prev return next_input
def init_training_graph(self): with tf.name_scope('Evaluation'): logits = self.last prob_b = tf.squeeze(logits, squeeze_dims=[1,2]) self.predictions = tf.argmax(prob_b, axis=1) with tf.name_scope('Loss'): self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prob_b, labels=tf.cast(self.train_labels_node, tf.int32), name="entropy"))) tf.summary.scalar("entropy", self.loss) with tf.name_scope('Accuracy'): LabelInt = tf.cast(self.train_labels_node, tf.int64) CorrectPrediction = tf.equal(self.predictions, LabelInt) self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32)) tf.summary.scalar("accuracy", self.accuracy) with tf.name_scope('Prediction'): self.TP = tf.count_nonzero(self.predictions * LabelInt) self.TN = tf.count_nonzero((self.predictions - 1) * (LabelInt - 1)) self.FP = tf.count_nonzero(self.predictions * (LabelInt - 1)) self.FN = tf.count_nonzero((self.predictions - 1) * LabelInt) with tf.name_scope('Precision'): self.precision = tf.divide(self.TP, tf.add(self.TP, self.FP)) tf.summary.scalar('Precision', self.precision) with tf.name_scope('Recall'): self.recall = tf.divide(self.TP, tf.add(self.TP, self.FN)) tf.summary.scalar('Recall', self.recall) with tf.name_scope('F1'): num = tf.multiply(self.precision, self.recall) dem = tf.add(self.precision, self.recall) self.F1 = tf.scalar_mul(2, tf.divide(num, dem)) tf.summary.scalar('F1', self.F1) with tf.name_scope('MeanAccuracy'): Nprecision = tf.divide(self.TN, tf.add(self.TN, self.FN)) self.MeanAcc = tf.divide(tf.add(self.precision, Nprecision) ,2) #self.batch = tf.Variable(0, name = "batch_iterator") self.train_prediction = tf.nn.softmax(logits) self.test_prediction = tf.nn.softmax(logits) tf.global_variables_initializer().run() print('Computational graph initialised')
def create_left_right(c, w, nrm): left = tf.transpose(tf.mul(tf.transpose(nrm), w)); right= tf.scalar_mul(-1.0, left); left = tf.add(left, c); right= tf.add(right,c); return left,right
def main(_): print('loading word embeddings from %s' % FLAGS.embedding_file) weight_matrix, word_idx = sentiment.load_embeddings(FLAGS.embedding_file) train_file = os.path.join(FLAGS.tree_dir, 'train.txt') print('loading training trees from %s' % train_file) train_trees = sentiment.load_trees(train_file) dev_file = os.path.join(FLAGS.tree_dir, 'dev.txt') print('loading dev trees from %s' % dev_file) dev_trees = sentiment.load_trees(dev_file) with tf.Session() as sess: print('creating the model') keep_prob = tf.placeholder_with_default(1.0, []) train_feed_dict = {keep_prob: FLAGS.keep_prob} word_embedding = sentiment.create_embedding(weight_matrix) compiler, metrics = sentiment.create_model( word_embedding, word_idx, FLAGS.lstm_num_units, keep_prob) loss = tf.reduce_sum(compiler.metric_tensors['all_loss']) opt = tf.train.AdagradOptimizer(FLAGS.learning_rate) grads_and_vars = opt.compute_gradients(loss) found = 0 for i, (grad, var) in enumerate(grads_and_vars): if var == word_embedding.weights: found += 1 grad = tf.scalar_mul(FLAGS.embedding_learning_rate_factor, grad) grads_and_vars[i] = (grad, var) assert found == 1 # internal consistency check train = opt.apply_gradients(grads_and_vars) saver = tf.train.Saver() print('initializing tensorflow') sess.run(tf.global_variables_initializer()) with compiler.multiprocessing_pool(): print('training the model') train_set = compiler.build_loom_inputs(train_trees) dev_feed_dict = compiler.build_feed_dict(dev_trees) dev_hits_best = 0.0 for epoch, shuffled in enumerate(td.epochs(train_set, FLAGS.epochs), 1): train_loss = 0.0 for batch in td.group_by_batches(shuffled, FLAGS.batch_size): train_feed_dict[compiler.loom_input_tensor] = batch _, batch_loss = sess.run([train, loss], train_feed_dict) train_loss += batch_loss dev_metrics = sess.run(metrics, dev_feed_dict) dev_loss = dev_metrics['all_loss'] dev_accuracy = ['%s: %.2f' % (k, v * 100) for k, v in sorted(dev_metrics.items()) if k.endswith('hits')] print('epoch:%4d, train_loss: %.3e, dev_loss: %.3e, dev_accuracy: [%s]' % (epoch, train_loss, dev_loss, ' '.join(dev_accuracy))) dev_hits = dev_metrics['root_hits'] if dev_hits > dev_hits_best: dev_hits_best = dev_hits save_path = saver.save(sess, FLAGS.checkpoint_base, global_step=epoch) print('model saved in file: %s' % save_path)
def _add_focal_losses(self, sigma_rpn=3.0): with tf.variable_scope('loss_' + self._tag) as scope: # RPN, class loss rpn_cls_score = tf.reshape(self._predictions['rpn_cls_score_reshape'], [-1, 2]) rpn_label = tf.reshape(self._anchor_targets['rpn_labels'], [-1]) rpn_select = tf.where(tf.not_equal(rpn_label, -1)) rpn_cls_score = tf.reshape(tf.gather(rpn_cls_score, rpn_select), [-1, 2]) rpn_label = tf.reshape(tf.gather(rpn_label, rpn_select), [-1]) rpn_cross_entropy = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits=rpn_cls_score, labels=rpn_label)) # RPN, bbox loss rpn_bbox_pred = self._predictions['rpn_bbox_pred'] rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets'] rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights'] rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights'] rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights, sigma=sigma_rpn, dim=[1, 2, 3]) # RCNN, class loss alpha_scale = 0.25 gamma = 2 epsilon = 1e-8 cls_score = self._predictions["cls_score"] label = tf.reshape(self._proposal_targets["labels"], [-1]) label = tf.one_hot(label, depth=self._num_classes) cls_pred = tf.nn.sigmoid(cls_score) predictions_pt = tf.where(tf.equal(label, 1), cls_pred, 1-cls_pred) alpha_t = tf.ones_like(label, dtype=tf.float32) alpha_t = tf.scalar_mul(alpha_scale, alpha_t) alpha_t = tf.where(tf.equal(label, 1.0), alpha_t, 1. - alpha_t) cross_entropy = tf.reduce_mean(-alpha_t*tf.pow(1 - predictions_pt, gamma)*tf.log(predictions_pt + epsilon)) # RCNN, bbox loss bbox_pred = self._predictions['bbox_pred'] bbox_targets = self._proposal_targets['bbox_targets'] bbox_inside_weights = self._proposal_targets['bbox_inside_weights'] bbox_outside_weights = self._proposal_targets['bbox_outside_weights'] loss_box = self._smooth_l1_loss(bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights) self._losses['cross_entropy'] = cross_entropy self._losses['loss_box'] = loss_box self._losses['rpn_cross_entropy'] = rpn_cross_entropy self._losses['rpn_loss_box'] = rpn_loss_box self._losses['rpn_loss'] = rpn_loss_box + rpn_cross_entropy self._losses['class_loss'] = cross_entropy + loss_box loss = cross_entropy + loss_box + rpn_cross_entropy + rpn_loss_box self._losses['total_loss'] = loss self._event_summaries.update(self._losses) return loss
def create_cost(center, xy, width, persitaltic, bend, bend_profiles, contour, contour_normals, length, weight_spacing = 1.0, weight_bending = 1.0, gamma = 1.0, kappa = 2.0): c = tf.add(center, xy); #tangent and normals t = create_tangent(c); tn = create_normalize_tangent(t); ta = create_average_tangent(tn); nl = create_normal(ta); #bend the center c = create_bend(center, nl, bend, bend_profiles); #peristaltically move the center c = create_peristaltic(c, ta, persitaltic); #tangents t = create_tangent(c); tn = create_normalize_tangent(t); ta = create_average_tangent(tn); nl = create_normal(ta); nr = tf.scalar_mul(-1.0, nl); l,r = create_left_right(c,width,nl); cost_left = create_cost_soft_min_aligned_distance(l, contour, nl, contour_normals, k = kappa, gamma = gamma); cost_right= create_cost_soft_min_aligned_distance(r, contour, nr, contour_normals, k = kappa, gamma = gamma); cost = tf.add(cost_left, cost_right); #spacing and bending if weight_spacing != 0: cost_spacing = tf.scalar_mul(weight_spacing, create_cost_spacing(t, length)); cost = tf.add(cost, cost_spacing); else: cost_spacing = tf.constant(0); if weight_bending != 0: cost_bending = tf.scalar_mul(weight_bending, create_cost_bending(tn)); cost = tf.add(cost, cost_bending); else: cost_bending = tf.constant(0); return (cost, cost_left, cost_right, cost_spacing, cost_bending, c, l, r, nl);
def focal_loss3(cls_score, label, num_classes): alpha_scale = 0.25 gamma = 2 epsilon = 1e-8 label = tf.one_hot(label, depth=num_classes) cls_pred = tf.nn.sigmoid(cls_score) predictions_pt = tf.where(tf.equal(label, 1), cls_pred, 1 - cls_pred) alpha_t = tf.ones_like(label, dtype=tf.float32) alpha_t = tf.scalar_mul(alpha_scale, alpha_t) alpha_t = tf.where(tf.equal(label, 1.0), alpha_t, 1. - alpha_t) losses = tf.reduce_mean(-alpha_t * tf.pow(1 - predictions_pt, gamma) * tf.log(predictions_pt + epsilon), axis=1) return losses
def build_model(self): self.images = tf.placeholder(tf.float32, [self.batch_size] + [self.output_size, self.output_size, self.c_dim], name='real_images') self.sample_images= tf.placeholder(tf.float32, [self.sample_size] + [self.output_size, self.output_size, self.c_dim], name='sample_images') self.z = tf.placeholder(tf.float32, [None, self.z_dim], name='z') self.z_sum = tf.summary.histogram("z", self.z) self.G = self.generator(self.z) self.D = self.discriminator(self.images) self.D_ = self.discriminator(self.G, reuse=True) self.sampler = self.sampler(self.z) self.d_sum = tf.summary.histogram("d", self.D) self.d__sum = tf.summary.histogram("d_", self.D_) self.G_sum = tf.summary.image("G", self.G) self.d_loss_real = tf.reduce_mean(tf.scalar_mul(-1, self.D)) self.d_loss_fake = tf.reduce_mean(self.D_) self.g_loss = tf.reduce_mean(tf.scalar_mul(-1, self.D_)) self.d_loss_real_sum = tf.summary.scalar("d_loss_real", self.d_loss_real) self.d_loss_fake_sum = tf.summary.scalar("d_loss_fake", self.d_loss_fake) self.d_loss = self.d_loss_real + self.d_loss_fake self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss) self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss) t_vars = tf.trainable_variables() self.d_vars = [var for var in t_vars if 'd_' in var.name] self.g_vars = [var for var in t_vars if 'g_' in var.name] self.epoch = tf.Variable(-1, name='epoch', trainable=False) self.increment_epoch = tf.assign(self.epoch, self.epoch+1) self.global_step = tf.Variable(0, name='global_step', trainable=False) self.saver = tf.train.Saver(max_to_keep=1000)
def entropy_matrix(graph, P): """ :param graph: :param P: :return: """ with graph.as_default(): shape = P.get_shape().as_list() one_diagonal = tf.diag(diagonal=[1.0 for _ in range(shape[0])]) P_mod = tf.add(P, one_diagonal) H = tf.reduce_sum(tf.scalar_mul(-1.0, tf.mul(P, tf.log(P_mod))), 1) return H
def _compute_update(self, param, grad, state): """Compute updates of parameters.""" # get the learning rate at the current index, if the index # is greater than the number of available learning rates, # use the last one index = tf.minimum(state["itr"], self.max_index) learning_rate = tf.gather(self.learning_rates, index) # update the parameters: parameter - learning_rate * gradient updated_param = param - tf.scalar_mul(learning_rate, grad) return updated_param, {"itr": state["itr"] + 1}
def init_training_graph(self): with tf.name_scope('Evaluation'): self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/") self.predictions = tf.argmax(self.logits, axis=3) with tf.name_scope('Loss'): self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=tf.squeeze(tf.cast(self.train_labels_node, tf.int32), squeeze_dims=[3]), name="entropy"))) tf.summary.scalar("entropy", self.loss) with tf.name_scope('Accuracy'): LabelInt = tf.squeeze(tf.cast(self.train_labels_node, tf.int64), squeeze_dims=[3]) CorrectPrediction = tf.equal(self.predictions, LabelInt) self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32)) tf.summary.scalar("accuracy", self.accuracy) with tf.name_scope('ClassPrediction'): flat_LabelInt = tf.reshape(LabelInt, [-1]) flat_predictions = tf.reshape(self.predictions, [-1]) self.cm = tf.confusion_matrix(flat_LabelInt, flat_predictions, self.NUM_LABELS) flatten_confusion_matrix = tf.reshape(self.cm, [-1]) total = tf.reduce_sum(self.cm) for i in range(self.NUM_LABELS): name = "Label_{}".format(i) TP, TN, FP, FN = GetCMInfo_TF(self.cm, i, self.NUM_LABELS) precision = tf.divide(TP, tf.add(TP, FP)) recall = tf.divide(TP, tf.add(TP, FN)) num = tf.multiply(precision, recall) dem = tf.add(precision, recall) F1 = tf.scalar_mul(2, tf.divide(num, dem)) Nprecision = tf.divide(TN, tf.add(TN, FN)) MeanAcc = tf.divide(tf.add(precision, Nprecision) ,2) tf.summary.scalar(name + '_Precision', precision) tf.summary.scalar(name + '_Recall', recall) tf.summary.scalar(name + '_F1', F1) tf.summary.scalar(name + '_Performance', MeanAcc) confusion_image = tf.reshape( tf.cast( self.cm, tf.float32), [1, self.NUM_LABELS, self.NUM_LABELS, 1]) tf.summary.image('confusion', confusion_image) self.train_prediction = tf.nn.softmax(self.logits) self.test_prediction = self.train_prediction tf.global_variables_initializer().run() print('Computational graph initialised')
def SimStep(x,In_Yr,In_Yp,In_k,In_dt): Yr=tf.constant(In_Yr,tf.float32)# Reactant ratios for each reaction Yp=tf.constant(In_Yp,tf.float32)# Product ratios for each reaction k=tf.constant(In_k,tf.float32)# Reaction constants for each reaction dt=tf.constant(In_dt,tf.float32) # time lapse for each simulation step s1=tf.pow(x,Yr) s2=tf.reduce_prod(s1,1) r=k*s2#Reacion rates s4=tf.scalar_mul(dt,r) Yd=Yp-Yr # Change in concentrations attribute to each reaction dxij=s4*tf.transpose(Yd)# concentration changes each reaction in this step*** dx=tf.reduce_sum(dxij,1) #sum of concentration changes from all reactions in this step*** xp=x+dx#New concentration after steps return(xp)
def __init__(self, x, weights: Weights, b: float, hp: Hyperparameters) -> None: self.x = tf.transpose(x) self.weights = weights self.b = b with tf.variable_scope('Learning'): self.alpha_ma_w = tf.scalar_mul(scalar=hp.alpha_ma, x=self.weights.w) self.one_minus_alpha_ma_x = tf.scalar_mul(scalar=1. - hp.alpha_ma, x=self.x) self.new_w = self.alpha_ma_w + self.one_minus_alpha_ma_x self.learn_weights_op = self.weights.assign(self.new_w) with tf.variable_scope('Layer'): with tf.variable_scope('Wx_plus_b'): z = tf.add(tf.matmul(x, weights.w), self.b) activation = tf.nn.relu(z, name='Activation') excitatory, inhibitory = weights.split(activation) self.excitatory_y = excitatory self.inhibitory_y = tf.scalar_mul(scalar=-1., x = inhibitory) self.y = tf.concat([self.excitatory_y, self.inhibitory_y], axis=1) util.variable_summaries(self.y) print(self.y, self.y.shape)
def AddGaussianNoise(t, sigma, noise_rate, name=None): """Add i.i.d. Gaussian noise (0, sigma^2) to every entry of t. Args: t: the input tensor. sigma: the stddev of the Gaussian noise. name: optional name. Returns: the noisy tensor. """ with tf.name_scope(values=[t, sigma], name=name, default_name="add_gaussian_noise") as name: noisy_t = t + tf.scalar_mul(noise_rate, tf.random_normal(tf.shape(t), stddev=sigma)) return noisy_t
def probability_matrix(graph, beta, D): """ :param D: :return: """ with graph.as_default(): shape = D.get_shape().as_list() beta_matrix = tf.tile(tf.expand_dims(beta, 1), multiples=[1, shape[1]]) sqrt_matrix = tf.tile(tf.constant(0.5, dtype=tf.float32, shape=[1,1]), multiples=[shape[0], shape[1]]) zero_diagonal = tf.exp(tf.diag(diagonal=[-np.inf for _ in range(shape[0])])) exp = tf.mul(tf.exp(tf.mul(beta_matrix, tf.scalar_mul(-1.0, tf.pow(D, sqrt_matrix)))), zero_diagonal) sum_exp = tf.reduce_sum(exp, reduction_indices=1) sum_exp_matrix = tf.tile(tf.expand_dims(sum_exp, 1), multiples=[1, shape[1]]) return tf.div(exp, sum_exp_matrix)
def calculate_euclidean_distance(training_data, test_data): # computing (x-y)^2 = x^2 + y^2 - 2xy first_term = tf.reduce_sum(training_data**2, axis=1) second_term = tf.expand_dims(tf.reduce_sum(test_data**2, axis=1), axis=1) xy = tf.matmul(test_data, tf.transpose(training_data)) third_term = tf.scalar_mul(-2, xy) dist = first_term + second_term + third_term # shape1 = tf.shape(first_term) # shape2 = tf.shape(second_term) # shape3 = tf.shape(third_term) # sess = tf.InteractiveSession() # print(sess.run([dist], feed_dict={X: trainData, Z: testData})) # print(sess.run([shape1, shape2, shape3], feed_dict={X: trainData, Z: testData})) return dist
def apply_weightshared_on_grads(grads_and_vars, dict_cenidx): # Mask gradients with pruned elements import config for key, cenidx in dict_cenidx.items(): count = 0 for grad, var in grads_and_vars: if var.name == key+":0": num_cen = config.kmeans_para[key] tem_grad=[] for i in range(num_cen): cenidx_obj = tf.cast(tf.constant(cenidx == i), tf.float32) tem_grad.append(tf.scalar_mul(tf.reduce_sum(tf.multiply(cenidx_obj,grad)),cenidx_obj)) grad=(tf.add_n(tem_grad)) grads_and_vars[count] = (grad, var) count += 1 return grads_and_vars
def get_batch(filename, batch_size): if not tf.gfile.Exists(filename): raise ValueError('Failed to find file ' + filename) # get single examples label, image = _read_and_decode_single_example(filename) # groups examples into batches randomly images_batch, labels_batch = tf.train.shuffle_batch( [image, label], batch_size=batch_size, capacity=2000, min_after_dequeue=1000) images_batch = tf.cast(images_batch, tf.float32) norm_factor = tf.constant(1.0/255) images_batch = tf.scalar_mul(norm_factor, images_batch) #labels_batch = tf.cast(labels_batch, tf.float32) return images_batch, labels_batch
def distance_matrix(graph, X): """ :param X: Input Matrix for which to calculate Distance Matrix :return: """ with graph.as_default(): shape = X.get_shape().as_list() Di = tf.reduce_sum(tf.mul(X, X), 1) I_magnitude_tiled = tf.tile(tf.expand_dims(Di, 1), multiples=[1, shape[0]]) I_t_magnitude_tiled = tf.tile(tf.expand_dims(Di, 0), multiples=[shape[0], 1]) I_I = tf.matmul(X, tf.transpose(X)) D = tf.add( tf.add(I_magnitude_tiled, I_t_magnitude_tiled), tf.scalar_mul(-2.0, I_I) ) zero_diagonal = tf.exp(tf.diag(diagonal=[-np.inf for _ in range(shape[0])])) return tf.mul(D, zero_diagonal)
def create_cost_soft_min_distance(self, c, s): """Creates a soft-min distance of the centers to the points""" c_shape = c.get_shape().as_list(); s_shape = s.get_shape().as_list(); #expand matrices cc = tf.reshape(c, [c_shape[0], c_shape[1], 1]); ss = tf.reshape(s, [s_shape[0], s_shape[1], 1]); ss = tf.transpose(ss, perm = [0,2,1]); cc = tf.tile(cc, [1, 1, s_shape[0]]); ss = tf.tile(ss, [c_shape[0], 1, 1]); #pairwise distances dist2 = tf.sqrt(tf.reduce_sum(tf.squared_difference(cc,ss), reduction_indices = 1)); dist2 = tf.reduce_mean(dist2, reduction_indices=0); # hack: get rid of batches here #softmin return tf.reduce_sum(tf.mul(tf.nn.softmax(tf.scalar_mul(tf.constant(-1.0,"float32"), dist2)), dist2),reduction_indices = 0);
def clip_norm(g, c, n): """Clip the gradient `g` if the L2 norm `n` exceeds `c`. # Arguments g: Tensor, the gradient tensor c: float >= 0. Gradients will be clipped when their L2 norm exceeds this value. n: Tensor, actual norm of `g`. # Returns Tensor, the gradient clipped if required. """ if c <= 0: # if clipnorm == 0 no need to add ops to the graph return g # tf require using a special op to multiply IndexedSliced by scalar if K.backend() == 'tensorflow': condition = n >= c then_expression = tf.scalar_mul(c / n, g) else_expression = g # saving the shape to avoid converting sparse tensor to dense if isinstance(then_expression, tf.Tensor): g_shape = copy.copy(then_expression.get_shape()) elif isinstance(then_expression, tf.IndexedSlices): g_shape = copy.copy(then_expression.dense_shape) if condition.dtype != tf.bool: condition = tf.cast(condition, 'bool') g = tf.cond(condition, lambda: then_expression, lambda: else_expression) if isinstance(then_expression, tf.Tensor): g.set_shape(g_shape) elif isinstance(then_expression, tf.IndexedSlices): g._dense_shape = g_shape else: g = K.switch(K.greater_equal(n, c), g * c / n, g) return g
def clip_norm(g, c, n): if c > 0: if K.backend() == 'tensorflow': import tensorflow as tf import copy condition = n >= c then_expression = tf.scalar_mul(c / n, g) else_expression = g if hasattr(then_expression, 'get_shape'): g_shape = copy.copy(then_expression.get_shape()) elif hasattr(then_expression, 'dense_shape'): g_shape = copy.copy(then_expression.dense_shape) if condition.dtype != tf.bool: condition = tf.cast(condition, 'bool') g = K.tensorflow_backend.control_flow_ops.cond( condition, lambda: then_expression, lambda: else_expression) if hasattr(then_expression, 'get_shape'): g.set_shape(g_shape) elif hasattr(then_expression, 'dense_shape'): g._dense_shape = g_shape else: g = K.switch(n >= c, g * c / n, g) return g
def train_setup(self): tf.set_random_seed(self.conf.random_seed) # Create queue coordinator. self.coord = tf.train.Coordinator() # Input size input_size = (self.conf.input_height, self.conf.input_width) # Load reader with tf.name_scope("create_inputs"): reader = ImageReader(self.conf.data_dir, self.conf.data_list, input_size, self.conf.random_scale, self.conf.random_mirror, self.conf.ignore_label, IMG_MEAN, self.coord) self.image_batch, self.label_batch = reader.dequeue( self.conf.batch_size) # Create network if self.conf.encoder_name not in ['res101', 'res50', 'deeplab']: print('encoder_name ERROR!') print("Please input: res101, res50, or deeplab") sys.exit(-1) elif self.conf.encoder_name == 'deeplab': net = Deeplab_v2(self.image_batch, self.conf.num_classes, True) # Variables that load from pre-trained model. restore_var = [ v for v in tf.global_variables() if 'fc' not in v.name ] # Trainable Variables all_trainable = tf.trainable_variables() # Fine-tune part encoder_trainable = [ v for v in all_trainable if 'fc' not in v.name ] # lr * 1.0 # Decoder part decoder_trainable = [v for v in all_trainable if 'fc' in v.name] else: net = ResNet_segmentation(self.image_batch, self.conf.num_classes, True, self.conf.encoder_name) # Variables that load from pre-trained model. restore_var = [ v for v in tf.global_variables() if 'resnet_v1' in v.name ] # Trainable Variables all_trainable = tf.trainable_variables() # Fine-tune part encoder_trainable = [ v for v in all_trainable if 'resnet_v1' in v.name ] # lr * 1.0 # Decoder part decoder_trainable = [ v for v in all_trainable if 'decoder' in v.name ] decoder_w_trainable = [ v for v in decoder_trainable if 'weights' in v.name or 'gamma' in v.name ] # lr * 10.0 decoder_b_trainable = [ v for v in decoder_trainable if 'biases' in v.name or 'beta' in v.name ] # lr * 20.0 # Check assert (len(all_trainable) == len(decoder_trainable) + len(encoder_trainable)) assert (len(decoder_trainable) == len(decoder_w_trainable) + len(decoder_b_trainable)) # Network raw output raw_output = net.outputs # [batch_size, h, w, 21] # Output size output_shape = tf.shape(raw_output) output_size = (output_shape[1], output_shape[2]) # Groud Truth: ignoring all labels greater or equal than n_classes label_proc = prepare_label(self.label_batch, output_size, num_classes=self.conf.num_classes, one_hot=False) raw_gt = tf.reshape(label_proc, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) raw_prediction = tf.reshape(raw_output, [-1, self.conf.num_classes]) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax_cross_entropy loss loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction, labels=gt) # L2 regularization l2_losses = [ self.conf.weight_decay * tf.nn.l2_loss(v) for v in all_trainable if 'weights' in v.name ] # Loss function self.reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) # Define optimizers # 'poly' learning rate base_lr = tf.constant(self.conf.learning_rate) self.curr_step = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - self.curr_step / self.conf.num_steps), self.conf.power)) # We have several optimizers here in order to handle the different lr_mult # which is a kind of parameters in Caffe. This controls the actual lr for each # layer. opt_encoder = tf.train.MomentumOptimizer(learning_rate, self.conf.momentum) opt_decoder_w = tf.train.MomentumOptimizer(learning_rate * 10.0, self.conf.momentum) opt_decoder_b = tf.train.MomentumOptimizer(learning_rate * 20.0, self.conf.momentum) # To make sure each layer gets updated by different lr's, we do not use 'minimize' here. # Instead, we separate the steps compute_grads+update_params. # Compute grads grads = tf.gradients( self.reduced_loss, encoder_trainable + decoder_w_trainable + decoder_b_trainable) grads_encoder = grads[:len(encoder_trainable)] grads_decoder_w = grads[len(encoder_trainable):( len(encoder_trainable) + len(decoder_w_trainable))] grads_decoder_b = grads[(len(encoder_trainable) + len(decoder_w_trainable)):] # Update params train_op_conv = opt_encoder.apply_gradients( zip(grads_encoder, encoder_trainable)) train_op_fc_w = opt_decoder_w.apply_gradients( zip(grads_decoder_w, decoder_w_trainable)) train_op_fc_b = opt_decoder_b.apply_gradients( zip(grads_decoder_b, decoder_b_trainable)) # Finally, get the train_op! update_ops = tf.get_collection( tf.GraphKeys.UPDATE_OPS ) # for collecting moving_mean and moving_variance with tf.control_dependencies(update_ops): self.train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) # Saver for storing checkpoints of the model self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=0) # Loader for loading the pre-trained model self.loader = tf.train.Saver(var_list=restore_var) # Training summary # Processed predictions: for visualisation. raw_output_up = tf.image.resize_bilinear(raw_output, input_size) raw_output_up = tf.argmax(raw_output_up, axis=3) self.pred = tf.expand_dims(raw_output_up, dim=3) # Image summary. images_summary = tf.py_func(inv_preprocess, [self.image_batch, 2, IMG_MEAN], tf.uint8) labels_summary = tf.py_func( decode_labels, [self.label_batch, 2, self.conf.num_classes], tf.uint8) preds_summary = tf.py_func(decode_labels, [self.pred, 2, self.conf.num_classes], tf.uint8) self.total_summary = tf.summary.image( 'images', tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]), max_outputs=2) # Concatenate row-wise. if not os.path.exists(self.conf.logdir): os.makedirs(self.conf.logdir) self.summary_writer = tf.summary.FileWriter( self.conf.logdir, graph=tf.get_default_graph())
def timeflow(self, t=0.0): wbs = tf.matmul(self.body.wb, self.body.Q) #[1,3] matrix tot_lbtomots = [] Feqc = tf.constant([0, 0, -Mtot * 9.81], dtype=tf.float32) Teqc = tf.zeros([1, 3], dtype=tf.float32) #List of External Forces Flist = [] llist = [] for p in range(numLeg): for i in range(numsubleg): #print('alpha = ',self.leg[p].sub[i].alpha); self.leg[p].sub[i].omega += self.leg[p].sub[ i].alpha * dtime #omega를 시간에 따라 갱신 #print('omega = ',self.leg[p].sub[i].omega); self.leg[p].sub[i].theta += self.leg[p].sub[ i].omega * dtime #theta를 시간에 따라 갱신 #print('theta = ',self.leg[p].sub[i].theta); self.leg[p].sub[i].Q = tf.scalar_mul(tf.cos(self.leg[p].sub[i].theta), tf.eye(3, dtype=tf.float32)) + \ tf.scalar_mul(1.-tf.cos(self.leg[p].sub[i].theta), tf.matmul(self.leg[p].sub[i].axis, self.leg[p].sub[i].axis, transpose_a = True)) + \ tf.scalar_mul(tf.sin(self.leg[p].sub[i].theta), tf.cross(tf.tile(self.leg[p].sub[i].axis,[3,1]), tf.eye(3, dtype=tf.float32))) Qs = [tf.matmul(self.leg[p].sub[0].Q, self.body.Q)] #Qs는 i번째 subleg에서 space로의 좌표변환 #List of rotation matrices of each sublegs in space frame #Type : list of [3,3] Tensor for i in range(1, numsubleg): Qs.append(tf.matmul(self.leg[p].sub[i].Q, Qs[i - 1])) e = [ tf.matmul(self.leg[p].sub[i].axis, Qs[i]) for i in range(numsubleg) ] #List of axes of each sublegs in space frame #Type : list of [None,3] Tensor Qw = [ tf.scalar_mul(self.leg[p].sub[i].omega, e[i]) for i in range(numsubleg) ] ws = [wbs + Qw[0]] for i in range(1, numsubleg): ws.append(ws[i - 1] + Qw[i]) ls = [[ tf.matmul(self.leg[p].sub[i].l[0], Qs[i]), tf.matmul(self.leg[p].sub[i].l[1], Qs[i]) ] for i in range(numsubleg)] #ls = 2Dtensor lbtomotbs = tf.matmul(self.body.lbtomot[p], self.body.Q) # lbtomotbs = 2Dtensor lbtomots = [lbtomotbs + ls[0][0]] # lbtomots = 2Dtensor for i in range(1, numsubleg): lbtomots.append(lbtomots[i - 1] + ls[i - 1][1] + ls[i][0]) #Calculating External Forces vstmp = self.body.vs + tf.cross(wbs, lbtomotbs) NormalScale = 300.0 TanhConst = 100.0 Zfilter = tf.constant([[0., 0., 1.]]) negZfilter = tf.constant([[0., 0., -1.]]) XYfilter = tf.constant([[1., 1., 0.]]) for i in range(numsubleg): Fz_primi = tf.multiply(negZfilter, self.body.rs + lbtomots[i] + ls[i][1]) colbool = tf.reshape( tf.matmul(Zfilter, tf.nn.relu(Fz_primi), transpose_b=True), []) Fz = tf.scalar_mul(NormalScale, tf.nn.relu(Fz_primi)) vstmp += tf.cross(ws[i], ls[i][0] + ls[i][1]) vstmp_z = tf.reshape( tf.matmul(Zfilter, vstmp, transpose_b=True), []) Vscale = 3. - tf.nn.tanh(TanhConst * vstmp_z) Fnormal = tf.scalar_mul(Vscale, Fz) Flist.append(Fnormal) vstmp_xy = tf.multiply(XYfilter, vstmp) Ffric = tf.scalar_mul(-Fricscale, tf.scalar_mul(colbool, vstmp_xy)) Flist.append(Ffric) Feqc += Fnormal Feqc += Ffric Teqc += tf.cross(lbtomots[i] + ls[i][1], Fnormal + Ffric) llist.append(tf.norm(lbtomots[i] + ls[i][1])) #Teqc+= tot_lbtomots += lbtomots asb = tf.scalar_mul(Mtotinv, Feqc) alphab = tf.matmul(tf.matmul(Teqc, self.body.Q, transpose_b=True), Ibinv) self.body.wb += tf.scalar_mul(dtime, alphab) self.body.Q += tf.scalar_mul( dtime, tf.cross(tf.concat([wbs, wbs, wbs], axis=0), self.body.Q)) self.body.vs += tf.scalar_mul(dtime, asb) self.body.rs += tf.scalar_mul(dtime, self.body.vs) # Q to quaternion ''' qw = tf.scalar_mul(0.5, tf.sqrt(tf.reduce_sum(tf.diag_part(self.body.Q))+1.)) qv = tf.reduce_sum(tf.cross(self.body.Q, tf.eye(3, dtype = tf.float32)), axis = 0)/tf.scalar_mul(4., qw) # quaternion normalization qvsquare = tf.reduce_sum(tf.square(qv)) qnorm = tf.sqrt(tf.square(qw)+qvsquare) qw /= qnorm qv /= qnorm # quaternion to Q self.body.Q = tf.scalar_mul(qw*qw-qvsquare,tf.eye(3, dtype = tf.float32))\ + 2 * tf.matmul(tf.reshape(qv, [3, 1]), tf.reshape(qv, [1, 3]))\ - 2 * qw * tf.cross(tf.tile(tf.reshape(qv, [1,3]), [3,1]), tf.eye(3, dtype = tf.float32)) ''' return llist, Flist, asb, Qs, [self.body.rs] + [ x + self.body.rs for x in tot_lbtomots ] #, qnorm, qw
def train(): """Train the Vnet model""" with tf.Graph().as_default(): global_step = tf.train.get_or_create_global_step() # patch_shape(batch_size, height, width, depth, channels) input_batch_shape = (FLAGS.batch_size, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1) output_batch_shape = (FLAGS.batch_size, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1) images_placeholder, labels_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape) for batch in range(FLAGS.batch_size): images_log = tf.cast(images_placeholder[batch:batch+1,:,:,:,0], dtype=tf.uint8) labels_log = tf.cast(tf.scalar_mul(255,labels_placeholder[batch:batch+1,:,:,:,0]), dtype=tf.uint8) tf.summary.image("image", tf.transpose(images_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer) tf.summary.image("label", tf.transpose(labels_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer) ''' import glob imagelist = glob.glob(os.path.join(FLAGS.data_dir,'Volume','*.nii.gz')) labellist = [imagename.replace('Volume','Label').replace('volume','label') for imagename in imagelist] imagetrainlist = imagelist[:int(len(imagelist)/2)] labeltrainlist = labellist[:int(len(imagelist)/2)] imagetestlist = imagelist[int(len(imagelist)/2):] labeltestlist = labellist[int(len(imagelist)/2):] ''' # Get images and labels train_data_dir = os.path.join(FLAGS.data_dir,'training') test_data_dir = os.path.join(FLAGS.data_dir,'testing') # support multiple image input, but here only use single channel, label file should be a single file with different classes image_filename = 'image_windowed.nii' label_filename = 'label.nii' # image_filename = 'img.nii.gz' # label_filename = 'label.nii.gz' # Force input pipepline to CPU:0 to avoid operations sometimes ended up at GPU and resulting a slow down with tf.device('/cpu:0'): # create transformations to image and labels trainTransforms = [ NiftiDataset.Normalization(), NiftiDataset.Resample(0.2), NiftiDataset.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)), NiftiDataset.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel), NiftiDataset.RandomNoise() ] TrainDataset = NiftiDataset.NiftiDataset( data_dir=train_data_dir, image_filename=image_filename, label_filename=label_filename, transforms=trainTransforms, train=True ) trainDataset = TrainDataset.get_dataset() trainDataset = trainDataset.shuffle(buffer_size=5) trainDataset = trainDataset.batch(FLAGS.batch_size) testTransforms = [ NiftiDataset.Normalization(), NiftiDataset.Resample(0.2), NiftiDataset.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)), NiftiDataset.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel) ] TestDataset = NiftiDataset.NiftiDataset( data_dir=test_data_dir, image_filename=image_filename, label_filename=label_filename, transforms=testTransforms, train=True ) testDataset = TestDataset.get_dataset() testDataset = testDataset.shuffle(buffer_size=5) testDataset = testDataset.batch(FLAGS.batch_size) train_iterator = trainDataset.make_initializable_iterator() next_element_train = train_iterator.get_next() test_iterator = testDataset.make_initializable_iterator() next_element_test = test_iterator.get_next() # Initialize the model with tf.name_scope("vnet"): logits = VNet.v_net(images_placeholder,input_channels = input_batch_shape[4], output_channels =2) # # apply weight to the logic funciton # if FLAGS.class_weight <0 or FLAGS.class_weight>1: # print("Class weight should between 0 and 1") # exit() # if logits.shape[4] == 2: # class_weight = tf.constant([FLAGS.class_weight,1.0-FLAGS.class_weight]) # weighted_logits = tf.multiply(logits,class_weight) # else: # weighted_logits = logits for batch in range(FLAGS.batch_size): logits_log_0 = tf.cast(logits[batch:batch+1,:,:,:,0], dtype=tf.uint8) logits_log_1 = tf.cast(logits[batch:batch+1,:,:,:,1], dtype=tf.uint8) tf.summary.image("logits_0", tf.transpose(logits_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer) tf.summary.image("logits_1", tf.transpose(logits_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer) # # Exponential decay learning rate # train_batches_per_epoch = math.ceil(TrainDataset.data_size/FLAGS.batch_size) # decay_steps = train_batches_per_epoch*FLAGS.decay_steps with tf.name_scope("learning_rate"): learning_rate = FLAGS.init_learning_rate # learning_rate = tf.train.exponential_decay(FLAGS.init_learning_rate, # global_step, # decay_steps, # FLAGS.decay_factor, # staircase=True) tf.summary.scalar('learning_rate', learning_rate) # softmax op for probability layer with tf.name_scope("softmax"): softmax_op = tf.nn.softmax(logits,name="softmax") for batch in range(FLAGS.batch_size): softmax_log_0 = tf.cast(tf.scalar_mul(255,softmax_op[batch:batch+1,:,:,:,0]), dtype=tf.uint8) softmax_log_1 = tf.cast(tf.scalar_mul(255,softmax_op[batch:batch+1,:,:,:,1]), dtype=tf.uint8) tf.summary.image("softmax_0", tf.transpose(softmax_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer) tf.summary.image("softmax_1", tf.transpose(softmax_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer) # Op for calculating loss with tf.name_scope("cross_entropy"): loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits, labels=tf.squeeze(labels_placeholder, squeeze_dims=[4]))) # loss_op = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy( # logits=logits, # labels=tf.squeeze(labels_placeholder,squeeze_dims=[4]), # weights=[[]])) tf.summary.scalar('loss',loss_op) # Argmax Op to generate label from logits with tf.name_scope("predicted_label"): pred = tf.argmax(logits, axis=4 , name="prediction") for batch in range(FLAGS.batch_size): pred_log = tf.cast(tf.scalar_mul(255,pred[batch:batch+1,:,:,:]), dtype=tf.uint8) tf.summary.image("pred", tf.transpose(pred_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer) # Accuracy of model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.expand_dims(pred,-1), tf.cast(labels_placeholder,dtype=tf.int64)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) tf.summary.scalar('accuracy', accuracy) # Dice Similarity with tf.name_scope("dice"): sorensen = dice_coe(tf.expand_dims(pred,-1),tf.cast(labels_placeholder,dtype=tf.int64), loss_type='sorensen') jaccard = dice_coe(tf.expand_dims(pred,-1),tf.cast(labels_placeholder,dtype=tf.int64), loss_type='jaccard') sorensen_loss = 1. - sorensen jaccard_loss = 1. - jaccard tf.summary.scalar('sorensen', sorensen) tf.summary.scalar('jaccard', jaccard) tf.summary.scalar('sorensen_loss', sorensen_loss) tf.summary.scalar('jaccard_loss',jaccard_loss) # Training Op with tf.name_scope("training"): # optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) optimizer = tf.train.GradientDescentOptimizer(learning_rate=FLAGS.init_learning_rate) train_op = optimizer.minimize( loss=loss_op, global_step=global_step) # # epoch checkpoint manipulation start_epoch = tf.get_variable("start_epoch", shape=[1], initializer= tf.zeros_initializer,dtype=tf.int32) start_epoch_inc = start_epoch.assign(start_epoch+1) # saver summary_op = tf.summary.merge_all() checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir ,"checkpoint") print("Setting up Saver...") saver = tf.train.Saver(keep_checkpoint_every_n_hours=5) # training cycle with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) print("{}: Start training...".format(datetime.datetime.now())) # summary writer for tensorboard train_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) test_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph) # restore from checkpoint if FLAGS.restore_training: # check if checkpoint exists if os.path.exists(checkpoint_prefix+"-latest"): print("{}: Last checkpoint found at {}, loading...".format(datetime.datetime.now(),FLAGS.checkpoint_dir)) latest_checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir,latest_filename="checkpoint-latest") saver.restore(sess, latest_checkpoint_path) print("{}: Last checkpoint epoch: {}".format(datetime.datetime.now(),start_epoch.eval()[0])) print("{}: Last checkpoint global step: {}".format(datetime.datetime.now(),tf.train.global_step(sess, global_step))) # loop over epochs for epoch in np.arange(start_epoch.eval(), FLAGS.epochs): # initialize iterator in each new epoch sess.run(train_iterator.initializer) sess.run(test_iterator.initializer) print("{}: Epoch {} starts".format(datetime.datetime.now(),epoch+1)) # training phase ''' reader = sitk.ImageFileReader() for imagepath, labelpath in zip(imagetrainlist,labeltrainlist): reader.SetFileName(imagepath) return reader.Execute() ''' while True: try: [image, label] = sess.run(next_element_train) image = image[:,:,:,:,np.newaxis] label = label[:,:,:,:,np.newaxis] train, train_loss, train_accuracy, train_jaccard, summary = sess.run( [train_op, loss_op, accuracy, jaccard, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label}) print("train loss: %.4f, accuracy: %.4f, jaccard: %.4f" % (train_loss, train_accuracy, train_jaccard)) train_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step)) except tf.errors.OutOfRangeError: start_epoch_inc.op.run() # print(start_epoch.eval()) # save the model at end of each epoch training print("{}: Saving checkpoint of epoch {} at {}...".format(datetime.datetime.now(),epoch+1,FLAGS.checkpoint_dir)) saver.save(sess, checkpoint_prefix, global_step=tf.train.global_step(sess, global_step), latest_filename="checkpoint-latest") print("{}: Saving checkpoint succeed".format(datetime.datetime.now())) break # testing phase print("{}: Training of epoch {} finishes, testing start".format(datetime.datetime.now(),epoch+1)) testJaccard = [] while True: try: [image, label] = sess.run(next_element_test) image = image[:,:,:,:,np.newaxis] label = label[:,:,:,:,np.newaxis] test_loss, test_accuracy, test_jaccard, summary = sess.run( [loss_op, accuracy, jaccard, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label}) print("test loss: %.4f, accuracy: %.4f, jaccard: %.4f" % (test_loss, test_accuracy, test_jaccard)) testJaccard.append(test_jaccard) #loss, summary = sess.run([loss_op, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label}) test_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step)) except tf.errors.OutOfRangeError: print("mean testing Jaccard: {}".format(np.array(testJaccard).mean())) break # close tensorboard summary writer train_summary_writer.close() test_summary_writer.close()
def apply_bernoulli_dropout(): if scale_during_training: return tf.nn.dropout(inference, keep_prob) else: return tf.scalar_mul(keep_prob,tf.nn.dropout(inference, keep_prob))
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) if args.center_crop_size is None: center_crop_size = None else: hc, wc = map(int, args.center_crop_size.split(',')) center_crop_size = (hc, wc) with tf.name_scope("create_inputs"): reader = ImageReader( DATA_DIR, DATA_LIST_PATH, input_size, center_crop_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN) image_batch, label_batch = reader.dequeue(args.batch_size) net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale) sub4_recls, sub24_recls, sub124_recls = bn_common.extend_reclassifier(net) restore_var = tf.global_variables() all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma] loss_sub4 = create_loss(sub4_recls, label_batch, args) loss_sub24 = create_loss(sub24_recls, label_batch, args) loss_sub124 = create_loss(sub124_recls, label_batch, args) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if ('weights' in v.name) or ('kernel' in v.name)] reduced_loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses) # print(tf.get_variable_scope().name) # print(','.join([v.__op.original_name_scope for v in l2_losses])) # print(','.join([v for v in tf.trainable_variables() if ('beta' in v.name or 'gamma' in v.name)])) # tf.summary.FileWriter('./summary', tf.get_default_graph()) # exit(0) # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) grads = tf.gradients(reduced_loss, all_trainable) train_op = opt_conv.apply_gradients(zip(grads, all_trainable)) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=99) ckpt = tf.train.get_checkpoint_state(args.snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load(loader, sess, ckpt.model_checkpoint_path) else: print('Restore from pre-trained model...') net.load(args.restore_from, sess) # Start queue threads. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) save(saver, sess, args.snapshot_dir, step) else: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration)) coord.request_stop() coord.join(threads) sess.close()
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) tf.set_random_seed(args.random_seed) # Create queue coordinator. coord = tf.train.Coordinator() # Load reader. with tf.name_scope("create_inputs"): reader = ImageReader( args.data_dir, args.data_list, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) # Create network. net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes) # For a small batch size, it is better to keep # the statistics of the BN layers (running means and variances) # frozen, and to not update the values provided by the pre-trained model. # If is_training=True, the statistics will be updated during the training. # Note that is_training=False still updates BN parameters gamma (scale) and beta (offset) # if they are presented in var_list of the optimiser definition. # Predictions. raw_output = net.layers['fc_out'] # Which variables to load. Running means and variances are not trainable, # thus all_variables() should be restored. restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last] all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name] fc_trainable = [v for v in all_trainable if 'fc' in v.name] conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0 fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0 fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0 assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable)) assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable)) # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, args.num_classes]) label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w] raw_gt = tf.reshape(label_proc, [-1,]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) # Processed predictions: for visualisation. raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Image summary. images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8) labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8) preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8) total_summary = tf.summary.image('images', tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]), max_outputs=args.save_num_images) # Concatenate row-wise. summary_writer = tf.summary.FileWriter(args.snapshot_dir, graph=tf.get_default_graph()) # Define loss and optimisation parameters. base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum) grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):] train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable)) train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10) ckpt = tf.train.get_checkpoint_state(SNAPSHOT_DIR) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') load_step = 0 # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = { step_ph : step } if step % args.save_pred_every == 0: loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, total_summary, train_op], feed_dict=feed_dict) summary_writer.add_summary(summary, step) save(saver, sess, args.snapshot_dir, step) else: loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration)) coord.request_stop() coord.join(threads)
def define(self): config = self.config with tf.Graph().as_default() as graph: self.graph = graph weights, biases = self.make_weights() self.training = tf.placeholder(tf.bool) # tf Graph Input self.X = X = tf.placeholder( tf.float32, [None, config['height'], config['width']]) if config['conv']: X1 = conv_net(X, config, weights['conv'], biases['conv'], config['conv']['max_pool_factors']) logits = fully_connected(X1, config, weights['fc'], biases['fc'], self.training) else: logits = fully_connected(X, config, weights['fc'], biases['fc'], self.training) n_classes = config['n_classes'] self.Y = Y = tf.placeholder(tf.int32, [None, config['n_outputs']]) Yhot = tf.one_hot(Y, n_classes) logits = tf.reshape(logits, [-1, config['n_outputs'], n_classes]) logit_loss = self.logit_loss(logits, Yhot) if config['l2_loss']: if config['conv']: l2_loss = tf.add_n( map(lambda weight: tf.nn.l2_loss(weight), weights['conv'])) else: l2_loss = tf.constant(0.) fc_l2_loss = tf.add_n( map(lambda weight: tf.nn.l2_loss(weight), weights['fc'])) l2_loss = tf.scalar_mul(config['l2_loss']['beta'], tf.add(l2_loss, fc_l2_loss)) else: l2_loss = tf.constant(0.) self.cost = cost = logit_loss #tf.add(logit_loss, l2_loss) tf.summary.scalar('cost', cost) self.pred = pred = tf.argmax(logits, axis=2) correct = tf.cast(tf.equal(tf.cast(pred, tf.int32), Y), tf.float32) correct_sequence = tf.cast( tf.equal(tf.reduce_sum(correct, 1), config['n_outputs']), tf.float32) self.accuracy = accuracy = tf.reduce_mean(correct_sequence) self.element_accuracy = element_accuracy = tf.reduce_mean(correct) tf.summary.scalar('set_accuracy', accuracy) tf.summary.scalar('element_accuracy', element_accuracy) self.global_step = global_step = tf.placeholder(tf.int32) if config['learning_rate']['decay']: learning_rate = tf.train.exponential_decay( config['learning_rate']['start'], global_step, config['iterations'], config['learning_rate']['coefficient']) else: learning_rate = tf.constant(config['learning_rate']['start']) tf.summary.scalar('learning_rate', learning_rate) if config['optimizer'] == 'Adam': self.optimizer = optimizer = tf.train.AdamOptimizer( learning_rate).minimize(logit_loss) elif config['optimizer'] == 'GradientDescent': self.optimizer = optimizer = tf.train.GradientDescentOptimizer( learning_rate).minimize(cost) # Initializing the variables self.init = tf.global_variables_initializer() self.summary = tf.summary.merge_all() dirname = 'output/%s_%s' % ( config['exp_name'], datetime.datetime.now().strftime('%Y_%m_%d_%H.%M')) self.train_writer = tf.summary.FileWriter(dirname, graph) # save hype for later comparisons with open(dirname + '/hypes.json', 'w') as f: json.dump(config, f, indent=2)
def compute_gradients(self, loss, var_list=None, gate_gradients=Optimizer.GATE_OP, aggregation_method=None, colocate_gradients_with_ops=False, grad_loss=None): """Compute gradients of `loss` for the variables in `var_list`. This is the first part of `minimize()`. It returns a list of (gradient, variable) pairs where "gradient" is the gradient for "variable". Note that "gradient" can be a `Tensor`, an `IndexedSlices`, or `None` if there is no gradient for the given variable. Args: loss: A Tensor containing the value to minimize or a callable taking no arguments which returns the value to minimize. When eager execution is enabled it must be a callable. var_list: Optional list or tuple of `tf.Variable` to update to minimize `loss`. Defaults to the list of variables collected in the graph under the key `GraphKeys.TRAINABLE_VARIABLES`. gate_gradients: How to gate the computation of gradients. Can be `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`. aggregation_method: Specifies the method used to combine gradient terms. Valid values are defined in the class `AggregationMethod`. colocate_gradients_with_ops: If True, try colocating gradients with the corresponding op. grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`. Returns: A list of (gradient, variable) pairs. Variable is always present, but gradient can be `None`. Raises: TypeError: If `var_list` contains anything else than `Variable` objects. ValueError: If some arguments are invalid. RuntimeError: If called with eager execution enabled and `loss` is not callable. @compatibility(eager) When eager execution is enabled, `gate_gradients`, `aggregation_method`, and `colocate_gradients_with_ops` are ignored. @end_compatibility """ if callable(loss): with backprop.GradientTape() as tape: if var_list is not None: tape.watch(var_list) loss_value = loss() if var_list is None: var_list = tape.watched_variables() # TODO(jhseu): Figure out why GradientTape's gradients don't require loss # to be executed. with ops.control_dependencies([loss_value]): grads = tape.gradient(loss_value, var_list, grad_loss) return list(zip(grads, var_list)) # # Non-callable/Tensor loss case # if context.executing_eagerly(): # raise RuntimeError( # "`loss` passed to Optimizer.compute_gradients should " # "be a function when eager execution is enabled.") if gate_gradients not in [SPSA.GATE_NONE, SPSA.GATE_OP, SPSA.GATE_GRAPH]: raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, " "Optimizer.GATE_OP, Optimizer.GATE_GRAPH. Not %s" % gate_gradients) self._assert_valid_dtypes([loss]) if grad_loss is not None: self._assert_valid_dtypes([grad_loss]) if var_list is None: var_list = ( variables.trainable_variables() + ops.get_collection(ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES)) else: var_list = nest.flatten(var_list) # pylint: disable=protected-access var_list += ops.get_collection(ops.GraphKeys._STREAMING_MODEL_PORTS) # pylint: enable=protected-access processors = [_get_processor(v) for v in var_list] if not var_list: raise ValueError("No variables to optimize.") var_refs = [p.target() for p in processors] # print("var_refs:") # for vr in var_refs: # print(vr) # ================================================================================== # grads = gradients.gradients( # loss, var_refs, grad_ys=grad_loss, # gate_gradients=(gate_gradients == SPSA.GATE_OP), # aggregation_method=aggregation_method, # colocate_gradients_with_ops=colocate_gradients_with_ops) # grads = [ tf.zeros(tf.shape(vrefs)) for vrefs in var_refs ] orig_graph_view = None trainable_vars = var_list # self.tvars = var_list self.tvars = [var.name.split(':')[0] for var in var_list] # list of names of trainable variables self.global_step_tensor = tf.Variable(0, name='global_step', trainable=False) # Perturbations deltas = {} n_perturbations = {} p_perturbations = {} with tf.name_scope("Perturbator"): self.c_t = tf.div( self.c, tf.pow(tf.add(tf.cast(self.global_step_tensor, tf.float32), tf.constant(1, dtype=tf.float32)), self.gamma), name = "SPSA_ct" ) for var in trainable_vars: self.num_params += self._mul_dims(var.get_shape()) var_name = var.name.split(':')[0] random = Bernoulli(tf.fill(var.get_shape(), 0.5), dtype=tf.float32) deltas[var] = tf.subtract( tf.constant(1, dtype=tf.float32), tf.scalar_mul(tf.constant(2, dtype=tf.float32),random.sample(1)[0]), name = "SPSA_delta" ) c_t_delta = tf.scalar_mul( tf.reshape(self.c_t, []), deltas[var] ) n_perturbations[var_name+'/read:0'] = tf.subtract( var, c_t_delta, name = "perturb_n" ) p_perturbations[var_name+'/read:0'] = tf.add(var, c_t_delta, name = "perturb_p" ) # print("{} parameters".format(self.num_params)) # Evaluator with tf.name_scope("Evaluator"): orig_graph_view = ge.sgv(tf.get_default_graph()) _, self.ninfo = self._clone_model(orig_graph_view, n_perturbations, 'N_Eval') _, self.pinfo = self._clone_model(orig_graph_view, p_perturbations, 'P_Eval') # Weight Updater optimizer_ops = [] grads = [] with tf.control_dependencies([loss]): with tf.name_scope('Updater'): a_t = self.a / (tf.pow(tf.add(tf.cast(self.global_step_tensor, tf.float32), tf.constant(1, dtype=tf.float32)), self.alpha)) for var in trainable_vars: l_pos = self.pinfo.transformed( loss ) l_neg = self.ninfo.transformed( loss ) ghat = (l_pos - l_neg) / (tf.constant(2, dtype=tf.float32) * self.c_t * deltas[var]) optimizer_ops.append(tf.assign_sub(var, a_t*ghat)) grads.append(ghat) print(tf.get_default_graph()) print("grads") for g in grads: print(g) #=================================================================================== if gate_gradients == SPSA.GATE_GRAPH: print("===================") grads = control_flow_ops.tuple(grads) grads_and_vars = list(zip(grads, var_list)) self._assert_valid_dtypes( [v for g, v in grads_and_vars if g is not None and v.dtype != dtypes.resource]) return grads_and_vars
def forward(self, reshaped_input): cluster_weights = tf.get_variable( "cluster_weights", [self.feature_size, self.cluster_size], initializer=tf.random_normal_initializer( stddev=1 / math.sqrt(self.feature_size))) covar_weights = tf.get_variable( "covar_weights", [self.feature_size, self.cluster_size], initializer=tf.random_normal_initializer( mean=1.0, stddev=1 / math.sqrt(self.feature_size))) covar_weights = tf.square(covar_weights) eps = tf.constant([1e-6]) covar_weights = tf.add(covar_weights, eps) tf.summary.histogram("cluster_weights", cluster_weights) activation = tf.matmul(reshaped_input, cluster_weights) if self.add_batch_norm: activation = slim.batch_norm(activation, center=True, scale=True, is_training=self.is_training, scope="cluster_bn") else: cluster_biases = tf.get_variable( "cluster_biases", [self.cluster_size], initializer=tf.random_normal(stddev=1 / math.sqrt(self.feature_size))) tf.summary.histogram("cluster_biases", cluster_biases) activation += cluster_biases activation = tf.nn.softmax(activation) tf.summary.histogram("cluster_output", activation) activation = tf.reshape(activation, [-1, self.max_frames, self.cluster_size]) a_sum = tf.reduce_sum(activation, -2, keep_dims=True) if not FLAGS.fv_couple_weights: cluster_weights2 = tf.get_variable( "cluster_weights2", [1, self.feature_size, self.cluster_size], initializer=tf.random_normal_initializer( stddev=1 / math.sqrt(self.feature_size))) else: cluster_weights2 = tf.scalar_mul(FLAGS.fv_coupling_factor, cluster_weights) a = tf.multiply(a_sum, cluster_weights2) activation = tf.transpose(activation, perm=[0, 2, 1]) reshaped_input = tf.reshape(reshaped_input, [-1, self.max_frames, self.feature_size]) fv1 = tf.matmul(activation, reshaped_input) fv1 = tf.transpose(fv1, perm=[0, 2, 1]) # computing second order FV a2 = tf.multiply(a_sum, tf.square(cluster_weights2)) b2 = tf.multiply(fv1, cluster_weights2) fv2 = tf.matmul(activation, tf.square(reshaped_input)) fv2 = tf.transpose(fv2, perm=[0, 2, 1]) fv2 = tf.add_n([a2, fv2, tf.scalar_mul(-2, b2)]) fv2 = tf.divide(fv2, tf.square(covar_weights)) fv2 = tf.subtract(fv2, a_sum) fv2 = tf.reshape(fv2, [-1, self.cluster_size * self.feature_size]) fv2 = tf.nn.l2_normalize(fv2, 1) fv2 = tf.reshape(fv2, [-1, self.cluster_size * self.feature_size]) fv2 = tf.nn.l2_normalize(fv2, 1) fv1 = tf.subtract(fv1, a) fv1 = tf.divide(fv1, covar_weights) fv1 = tf.nn.l2_normalize(fv1, 1) fv1 = tf.reshape(fv1, [-1, self.cluster_size * self.feature_size]) fv1 = tf.nn.l2_normalize(fv1, 1) return tf.concat([fv1, fv2], 1)
def main(parsed_arguments): tf.enable_eager_execution() if not isinstance(parsed_arguments.threads, int): parsed_arguments.threads = int(parsed_arguments.threads) try: architecture_json_filename = parsed_arguments.json_filename architecture_json_content = open(architecture_json_filename, 'r').read() parsed_architecture_json = json.loads(architecture_json_content) except: print('Expected a valid architecture json file.') assert os.path.isdir(parsed_arguments.input) if not isinstance(parsed_arguments.tile_size, int): parsed_arguments.tile_size = int(parsed_arguments.tile_size) if not isinstance(parsed_arguments.tile_overlap_size, int): parsed_arguments.tile_overlap_size = int( parsed_arguments.tile_overlap_size) tile_size = parsed_arguments.tile_size tile_overlap_size = parsed_arguments.tile_overlap_size data_format = parsed_arguments.data_format architecture = Architecture(parsed_architecture_json, source_data_format='channels_last', data_format=data_format) if architecture.data_format == 'channels_first': use_CPU_only = False else: use_CPU_only = True height = None width = None exr_files = OpenEXRDirectory._exr_files(parsed_arguments.input) features = {} required_features = architecture.auxiliary_features + architecture.feature_predictions for feature_prediction in required_features: exr_loaded = False if feature_prediction.load_data: for exr_file in exr_files: if feature_prediction.name in exr_file: image = OpenEXRDirectory._load_exr(exr_file) # HACK: Assume just one source input! features[Naming.source_feature_name( feature_prediction.name, index=0)] = image exr_loaded = True if height == None: height = image.shape[0] width = image.shape[1] else: assert height == image.shape[0] assert width == image.shape[1] break else: image = tf.ones( [height, width, feature_prediction.number_of_channels]) if feature_prediction.feature_prediction_type != FeaturePredictionType.COLOR: # Direct and indirect need to be 0.5. image = tf.scalar_mul(0.5, image) features[Naming.source_feature_name(feature_prediction.name, index=0)] = image exr_loaded = True if not exr_loaded: # TODO: Improve (DeepBlender) raise Exception('Image for \'' + feature_prediction.name + '\' could not be loaded or does not exist.') smaller_side_length = min(height, width) if smaller_side_length < 16: raise Exception( 'The image needs to have at least a side length of 16 pixels.') if smaller_side_length < tile_size: ratio = tile_overlap_size / tile_size tile_size = smaller_side_length tile_overlap_size = int(tile_size * ratio) # Split the images into tiles. iteration_delta = tile_size - (2 * tile_overlap_size) width_count = width - (2 * tile_overlap_size) - (2 * iteration_delta) width_count = width_count / iteration_delta width_count = math.ceil(width_count) + 2 height_count = height - (2 * tile_overlap_size) - (2 * iteration_delta) height_count = height_count / iteration_delta height_count = math.ceil(height_count) + 2 tiled_features_grid = [[None for _ in range(width_count)] for _ in range(height_count)] for height_index in range(height_count): if height_index == 0: lower_height = 0 upper_height = tile_size elif height_index == height_count - 1: upper_height = height lower_height = upper_height - tile_size else: lower_height = height_index * iteration_delta upper_height = lower_height + tile_size for width_index in range(width_count): if width_index == 0: lower_width = 0 upper_width = tile_size elif width_index == width_count - 1: upper_width = width lower_width = upper_width - tile_size else: lower_width = width_index * iteration_delta upper_width = lower_width + tile_size tiled_features = {} for feature_name in features: feature = features[feature_name] tiled_feature = feature[lower_height:upper_height, lower_width:upper_width] tiled_features[feature_name] = tiled_feature tiled_features_grid[height_index][width_index] = tiled_features # We don't need the features anymore. features = None if use_CPU_only: session_config = tf.ConfigProto(device_count={'GPU': 0}) else: session_config = tf.ConfigProto() use_XLA = True if use_XLA: session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 run_config = tf.estimator.RunConfig(session_config=session_config) estimator = tf.estimator.Estimator(model_fn=model_fn, model_dir=architecture.model_directory, config=run_config, params={'architecture': architecture}) tiled_features_list = [] for height_index in range(height_count): for width_index in range(width_count): tiled_features = tiled_features_grid[height_index][width_index] tiled_features_list.append(tiled_features) predictions = estimator.predict(input_fn=lambda: input_fn_predict( tiled_features_list, tile_size, tile_size)) for height_index in range(height_count): for width_index in range(width_count): tiled_features_grid[height_index][width_index] = next(predictions) predictions = {} for feature_prediction_tuple in architecture.feature_prediction_tuples: for feature_prediction in feature_prediction_tuple.feature_predictions: if feature_prediction.load_data: horizontal_feature_stripes = [] for height_index in range(height_count): horizontal_feature_elements = [] for width_index in range(width_count): tiled_predictions = tiled_features_grid[height_index][ width_index] prediction_name = Naming.feature_prediction_name( feature_prediction.name) prediction = tiled_predictions[prediction_name] lower_height = 0 upper_height = tile_size lower_width = 0 upper_width = tile_size if width_index != 0 and width_index != width_count - 1: lower_width = tile_overlap_size upper_width = upper_width - tile_overlap_size elif width_index == 0 and width_index == width_count - 1: pass elif width_index == 0: upper_width = upper_width - tile_overlap_size else: assert width_index == width_count - 1 existing_width = tile_overlap_size + ( (width_count - 1) * (tile_size - (2 * tile_overlap_size))) remaining_width = width - existing_width lower_width = upper_width - remaining_width if height_index != 0 and height_index != height_count - 1: lower_height = tile_overlap_size upper_height = upper_height - tile_overlap_size elif height_index == 0 and height_index == height_count - 1: pass elif height_index == 0: upper_height = upper_height - tile_overlap_size else: assert height_index == height_count - 1 existing_height = tile_overlap_size + ( (height_count - 1) * (tile_size - (2 * tile_overlap_size))) remaining_height = height - existing_height lower_height = upper_height - remaining_height prediction = prediction[lower_height:upper_height, lower_width:upper_width] horizontal_feature_elements.append(prediction) if len(horizontal_feature_elements) > 1: horizontal_feature_stripe = np.concatenate( horizontal_feature_elements, 1) else: horizontal_feature_stripe = horizontal_feature_elements[ 0] horizontal_feature_stripes.append( horizontal_feature_stripe) if len(horizontal_feature_stripes) > 1: prediction = np.concatenate(horizontal_feature_stripes, 0) else: prediction = horizontal_feature_stripes[0] prediction_name = Naming.feature_prediction_name( feature_prediction.name) predictions[prediction_name] = prediction diffuse_direct = predictions[Naming.feature_prediction_name( RenderPasses.DIFFUSE_DIRECT)] diffuse_indirect = predictions[Naming.feature_prediction_name( RenderPasses.DIFFUSE_INDIRECT)] diffuse_color = predictions[Naming.feature_prediction_name( RenderPasses.DIFFUSE_COLOR)] glossy_direct = predictions[Naming.feature_prediction_name( RenderPasses.GLOSSY_DIRECT)] glossy_indirect = predictions[Naming.feature_prediction_name( RenderPasses.GLOSSY_INDIRECT)] glossy_color = predictions[Naming.feature_prediction_name( RenderPasses.GLOSSY_COLOR)] subsurface_direct = predictions[Naming.feature_prediction_name( RenderPasses.SUBSURFACE_DIRECT)] subsurface_indirect = predictions[Naming.feature_prediction_name( RenderPasses.SUBSURFACE_INDIRECT)] subsurface_color = predictions[Naming.feature_prediction_name( RenderPasses.SUBSURFACE_COLOR)] transmission_direct = predictions[Naming.feature_prediction_name( RenderPasses.TRANSMISSION_DIRECT)] transmission_indirect = predictions[Naming.feature_prediction_name( RenderPasses.TRANSMISSION_INDIRECT)] transmission_color = predictions[Naming.feature_prediction_name( RenderPasses.TRANSMISSION_COLOR)] volume_direct = predictions[Naming.feature_prediction_name( RenderPasses.VOLUME_DIRECT)] volume_indirect = predictions[Naming.feature_prediction_name( RenderPasses.VOLUME_INDIRECT)] environment = predictions[Naming.feature_prediction_name( RenderPasses.ENVIRONMENT)] emission = predictions[Naming.feature_prediction_name( RenderPasses.EMISSION)] # Combined features diffuse = np.multiply(diffuse_color, np.add(diffuse_direct, diffuse_indirect)) glossy = np.multiply(glossy_color, np.add(glossy_direct, glossy_indirect)) subsurface = np.multiply(subsurface_color, np.add(subsurface_direct, subsurface_indirect)) transmission = np.multiply( transmission_color, np.add(transmission_direct, transmission_indirect)) # Combined image image = np.add(diffuse, glossy) image = np.add(image, subsurface) image = np.add(image, transmission) image = np.add(image, volume_direct) image = np.add(image, volume_indirect) image = np.add(image, environment) image = np.add(image, emission) # TODO: Alpha currently ignored. # Store as npy to open in Blender. np.save(parsed_arguments.input + '/combined.npy', image) # HACK: Temporary output as png. (DeepBlender) image = 255. * image image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) cv2.imwrite(parsed_arguments.input + '/combined.png', image, [int(cv2.IMWRITE_PNG_COMPRESSION), 9])
dense_shape=[2, 4]) mat = tf.sparse.SparseTensor( indices=[[1, 0], [2, 0], [3, 0], [3, 1], [0, 2], [1, 3]], values=[0.85 * 0.33, 0.85 * 0.33, 0.85 * 0.33, 0.85, 0.85, 0.85], dense_shape=[4, 4]) neg_beta = 0.15 / 4 prev_rank = tf.get_variable("prev_rank", dtype=float, initializer=tf.constant([[1 / 4] for _ in range(4)])) curr_rank = tf.get_variable("curr_rank", dtype=float, initializer=tf.constant([[1 / 4] for _ in range(4)])) i = 0 with tf.Session() as sess: sess.run(init_op) while True: print(i) i += 1 prev_rank = curr_rank curr_rank = tf.add( tf.sparse.sparse_dense_matmul(mat, prev_rank), tf.fill(dims=[4, 1], value=tf.reduce_sum(tf.scalar_mul(neg_beta, prev_rank)))) tf.global_variables_initializer().run() #print(sess.run(curr_rank)) if sess.run(tf.reduce_sum(tf.abs(curr_rank - prev_rank))) < 0.00001: print("Here") print(sess.run(curr_rank)) break
def train(): global_step = tf.Variable(0, name = 'global_step', trainable = False) train_dir = CURRENT_DIR + '/logs_without_condition/' data_dir = CURRENT_DIR + '/data/img_align_celeba_tfrecords/' images = inputs(data_dir, BATCH_SIZE) z = tf.placeholder(tf.float32, [None, Z_DIM], name='z') G = generator(z) _, D_logits = discriminator(images) samples = sampler(z) _, D_logits_ = discriminator(G, reuse = True) d_loss_real = tf.reduce_mean(tf.scalar_mul(-1, D_logits)) d_loss_fake = tf.reduce_mean(D_logits_) d_loss = d_loss_real + d_loss_fake g_loss = tf.reduce_mean(tf.scalar_mul(-1, D_logits_)) z_sum = tf.summary.histogram('z', z) d_sum = tf.summary.histogram('d', D_logits) d__sum = tf.summary.histogram('d_', D_logits_) G_sum = tf.summary.image('G', G) d_loss_real_sum = tf.summary.scalar('d_loss_real', d_loss_real) d_loss_fake_sum = tf.summary.scalar('d_loss_fake', d_loss_fake) d_loss_sum = tf.summary.scalar('d_loss', d_loss) g_loss_sum = tf.summary.scalar('g_loss', g_loss) g_sum = tf.summary.merge([z_sum, d__sum, G_sum, d_loss_fake_sum, g_loss_sum]) d_sum = tf.summary.merge([z_sum, d_sum, d_loss_real_sum, d_loss_sum]) t_vars = tf.trainable_variables() d_vars = [var for var in t_vars if 'd_' in var.name] g_vars = [var for var in t_vars if 'g_' in var.name] saver = tf.train.Saver() with tf.variable_scope("", reuse=tf.AUTO_REUSE): d_optim = tf.train.RMSPropOptimizer(LR).minimize(d_loss, var_list = d_vars, global_step = global_step) g_optim = tf.train.RMSPropOptimizer(LR).minimize(g_loss, var_list = g_vars, global_step = global_step) clip_d_op = [var.assign(tf.clip_by_value(var, CLIP[0], CLIP[1])) for var in d_vars] #os.environ['CUDA_VISIBLE_DEVICES'] = str(0) #config = tf.ConfigProto() #config.gpu_options.per_process_gpu_memory_fraction = 0.2 #sess = tf.InteractiveSession(config=config) sess = tf.Session() writer = tf.summary.FileWriter(train_dir, sess.graph) sample_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM)) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess = sess, coord = coord) init = tf.initialize_all_variables() sess.run(init) start = 0 if LOAD_MODEL: print(" [*] Reading checkpoints...") ckpt = tf.train.get_checkpoint_state(train_dir) if ckpt and ckpt.model_checkpoint_path: ckpt_name = os.path.basename(ckpt.model_checkpoint_path) saver.restore(sess, os.path.join(train_dir, ckpt_name)) global_step = ckpt.model_checkpoint_path.split('/')[-1]\ .split('-')[-1] print('Loading success, global_step is %s' % global_step) start = int(global_step) for epoch in range(EPOCH): batch_idxs = 3072 if epoch: start = 0 for idx in range(start, batch_idxs): if idx<25 or idx % 500 == 0: critic_num = 25 else: critic_num = CRITIC_NUM for _ in range(critic_num): batch_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM)) _,summary_str = sess.run([d_optim,d_sum], feed_dict = {z:batch_z}) sess.run(clip_d_op) writer.add_summary(summary_str,idx+1) batch_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM)) _, summary_str = sess.run([d_optim, d_sum], feed_dict = {z: batch_z}) writer.add_summary(summary_str, idx+1) # Update G network _, summary_str = sess.run([g_optim, g_sum], feed_dict = {z: batch_z}) writer.add_summary(summary_str, idx+1) # Run g_optim twice to make sure that d_loss does not go to zero _, summary_str = sess.run([g_optim, g_sum], feed_dict = {z: batch_z}) writer.add_summary(summary_str, idx+1) errD_fake = 0#d_loss_fake.eval({z: batch_z}) errD_real = 0#d_loss_real.eval() errG = 0#g_loss.eval({z: batch_z}) if idx % 20 == 0: print("[%4d/%4d] d_loss: %.8f, g_loss: %.8f" \ % (idx, batch_idxs, errD_fake+errD_real, errG)) if idx % 100 == 0: sample = sess.run(samples, feed_dict = {z: sample_z}) samples_path = CURRENT_DIR + '\\out\\' save_images(sample, [8, 8], samples_path + \ 'sample_%d_epoch_%d.png' % (epoch, idx)) print('\n'*2) print('=========== %d_epoch_%d.png save down ===========' %(epoch, idx)) print('\n'*2) if (idx % 512 == 0) or (idx + 1 == batch_idxs): checkpoint_path = os.path.join(train_dir, 'my_dcgan_tfrecords.ckpt') saver.save(sess, checkpoint_path, global_step = idx+1) print('********* model saved *********') print('******* start with %d *******' % start) coord.request_stop() coord.join(threads) sess.close()
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) tf.set_random_seed(args.random_seed) # Create queue coordinator. coord = tf.train.Coordinator() # Load reader. with tf.name_scope("create_inputs"): reader = ImageReader(args.data_dir, args.data_list, input_size, args.random_scale, args.random_mirror, coord) image_batch, label_batch = reader.dequeue(args.batch_size) image_batch075 = tf.image.resize_images( image_batch, [int(h * 0.75), int(w * 0.75)]) image_batch05 = tf.image.resize_images( image_batch, [int(h * 0.5), int(w * 0.5)]) # Create network. with tf.variable_scope('', reuse=False): net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training) with tf.variable_scope('', reuse=True): net075 = DeepLabResNetModel({'data': image_batch075}, is_training=args.is_training) with tf.variable_scope('', reuse=True): net05 = DeepLabResNetModel({'data': image_batch05}, is_training=args.is_training) # For a small batch size, it is better to keep # the statistics of the BN layers (running means and variances) # frozen, and to not update the values provided by the pre-trained model. # If is_training=True, the statistics will be updated during the training. # Note that is_training=False still updates BN parameters gamma (scale) and beta (offset) # if they are presented in var_list of the optimiser definition. # Predictions. raw_output100 = net.layers['fc1_voc12'] raw_output075 = net075.layers['fc1_voc12'] raw_output05 = net05.layers['fc1_voc12'] raw_output = tf.reduce_max(tf.stack([ raw_output100, tf.image.resize_images(raw_output075, tf.shape(raw_output100)[1:3, ]), tf.image.resize_images(raw_output05, tf.shape(raw_output100)[1:3, ]) ]), axis=0) # Which variables to load. Running means and variances are not trainable, # thus all_variables() should be restored. restore_var = tf.global_variables() all_trainable = [ v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name ] fc_trainable = [v for v in all_trainable if 'fc' in v.name] conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0 fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0 fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0 assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable)) assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable)) # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, n_classes]) raw_prediction100 = tf.reshape(raw_output100, [-1, n_classes]) raw_prediction075 = tf.reshape(raw_output075, [-1, n_classes]) raw_prediction05 = tf.reshape(raw_output05, [-1, n_classes]) label_proc = prepare_label(label_batch, tf.pack(raw_output.get_shape()[1:3]), one_hot=False) # [batch_size, h, w] label_proc075 = prepare_label(label_batch, tf.pack(raw_output075.get_shape()[1:3]), one_hot=False) label_proc05 = prepare_label(label_batch, tf.pack(raw_output05.get_shape()[1:3]), one_hot=False) raw_gt = tf.reshape(label_proc, [ -1, ]) raw_gt075 = tf.reshape(label_proc075, [ -1, ]) raw_gt05 = tf.reshape(label_proc05, [ -1, ]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, n_classes - 1)), 1) indices075 = tf.squeeze(tf.where(tf.less_equal(raw_gt075, n_classes - 1)), 1) indices05 = tf.squeeze(tf.where(tf.less_equal(raw_gt05, n_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32) gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32) prediction = tf.gather(raw_prediction, indices) prediction100 = tf.gather(raw_prediction100, indices) prediction075 = tf.gather(raw_prediction075, indices075) prediction05 = tf.gather(raw_prediction05, indices05) # Pixel-wise softmax loss. loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction100, labels=gt) loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction075, labels=gt075) loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=prediction05, labels=gt05) l2_losses = [ args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name ] reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean( loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(loss05) + tf.add_n( l2_losses) # Processed predictions: for visualisation. raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Image summary. images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images], tf.uint8) labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images], tf.uint8) preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images], tf.uint8) total_summary = tf.summary.image( 'images', tf.concat(2, [images_summary, labels_summary, preds_summary]), max_outputs=args.save_num_images) # Concatenate row-wise. summary_writer = tf.summary.FileWriter(args.snapshot_dir, graph=tf.get_default_graph()) # Define loss and optimisation parameters. base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum) # Define a variable to accumulate gradients. accum_grads = [ tf.Variable(tf.zeros_like(v.initialized_value()), trainable=False) for v in conv_trainable + fc_w_trainable + fc_b_trainable ] # Define an operation to clear the accumulated gradients for next batch. zero_op = [v.assign(tf.zeros_like(v)) for v in accum_grads] # Compute gradients. grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) # Accumulate and normalise the gradients. accum_grads_op = [ accum_grads[i].assign_add(grad / args.grad_update_every) for i, grad in enumerate(grads) ] grads_conv = accum_grads[:len(conv_trainable)] grads_fc_w = accum_grads[len(conv_trainable):(len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = accum_grads[(len(conv_trainable) + len(fc_w_trainable)):] # Apply the gradients. train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable)) train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b) # Set up tf session and initialize variables. config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=restore_var, max_to_keep=10) # Load variables if the checkpoint is provided. if args.restore_from is not None: loader = tf.train.Saver(var_list=restore_var) load(loader, sess, args.restore_from) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} loss_value = 0 # Clear the accumulated gradients. sess.run(zero_op, feed_dict=feed_dict) # Accumulate gradients. for i in range(args.grad_update_every): _, l_val = sess.run([accum_grads_op, reduced_loss], feed_dict=feed_dict) loss_value += l_val # Normalise the loss. loss_value /= args.grad_update_every # Apply gradients. if step % args.save_pred_every == 0: images, labels, summary, _ = sess.run( [image_batch, label_batch, total_summary, train_op], feed_dict=feed_dict) summary_writer.add_summary(summary, step) save(saver, sess, args.snapshot_dir, step) else: sess.run(train_op, feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format( step, loss_value, duration)) coord.request_stop() coord.join(threads)
def train(target, dataset, cluster_spec): """Train Inception on a dataset for a number of steps.""" # Number of workers and parameter servers are inferred from the workers and ps # hosts string. num_workers = len(cluster_spec.as_dict()['worker']) num_parameter_servers = len(cluster_spec.as_dict()['ps']) # If no value is given, num_replicas_to_aggregate defaults to be the number of # workers. if FLAGS.num_replicas_to_aggregate == -1: num_replicas_to_aggregate = num_workers else: num_replicas_to_aggregate = FLAGS.num_replicas_to_aggregate # Both should be greater than 0 in a distributed training. assert num_workers > 0 and num_parameter_servers > 0, (' num_workers and ' 'num_parameter_servers' ' must be > 0.') # Choose worker 0 as the chief. Note that any worker could be the chief # but there should be only one chief. is_chief = (FLAGS.task_id == 0) #batchSizeManager = BatchSizeManager(32, 4) # Ops are assigned to worker by default. tf.logging.info('cccc-num_parameter_servers:'+str(num_parameter_servers)) partitioner = tf.fixed_size_partitioner(num_parameter_servers, 0) device_setter = tf.train.replica_device_setter(ps_tasks=num_parameter_servers) slim = tf.contrib.slim with tf.device('/job:worker/task:%d' % FLAGS.task_id): with tf.variable_scope('root', partitioner=partitioner): # Variables and its related init/assign ops are assigned to ps. # with slim.arg_scope( # [slim.variables.variable, slim.variables.global_step], # device=slim.variables.VariableDeviceChooser(num_parameter_servers)): with tf.device(device_setter): # partitioner=partitioner): # Create a variable to count the number of train() calls. This equals the # number of updates applied to the variables. # global_step = slim.variables.global_step() global_step = tf.Variable(0, trainable=False) # Calculate the learning rate schedule. batch_size = tf.placeholder(dtype=tf.int32, shape=(), name='batch_size') num_batches_per_epoch = (dataset.num_examples_per_epoch() / FLAGS.batch_size) # Decay steps need to be divided by the number of replicas to aggregate. decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay / num_replicas_to_aggregate) # Decay the learning rate exponentially based on the number of steps. lr = tf.train.exponential_decay(FLAGS.initial_learning_rate, global_step, decay_steps, FLAGS.learning_rate_decay_factor, staircase=True) # Add a summary to track the learning rate. # tf.summary.scalar('learning_rate', lr) # Create an optimizer that performs gradient descent. opt = tf.train.RMSPropOptimizer(lr, RMSPROP_DECAY, momentum=RMSPROP_MOMENTUM, epsilon=RMSPROP_EPSILON) images, labels = image_processing.distorted_inputs( dataset, batch_size, num_preprocess_threads=FLAGS.num_preprocess_threads) print(images.get_shape()) print(labels.get_shape()) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. # num_classes = dataset.num_classes() + 1 num_classes = dataset.num_classes() print(num_classes) # logits = inception.inference(images, num_classes, for_training=True) network_fn = nets_factory.get_network_fn('inception_v3',num_classes=num_classes) (logits,_) = network_fn(images) print(logits.get_shape()) # Add classification loss. # inception.loss(logits, labels, batch_size) # Gather all of the losses including regularization losses. labels = tf.one_hot(labels, 1000, 1, 0) cross_entropy = tf.losses.softmax_cross_entropy( logits=logits, onehot_labels=labels) # losses = tf.get_collection(slim.losses.LOSSES_COLLECTION) # losses += tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) total_loss = cross_entropy + _WEIGHT_DECAY * tf.add_n( [tf.nn.l2_loss(v) for v in tf.trainable_variables()]) # total_loss = tf.add_n(losses, name='total_loss') if is_chief: # Compute the moving average of all individual losses and the # total loss. loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg') loss_averages_op = loss_averages.apply(losses + [total_loss]) # Attach a scalar summmary to all individual losses and the total loss; # do the same for the averaged version of the losses. # for l in losses + [total_loss]: # loss_name = l.op.name # Name each loss as '(raw)' and name the moving average version of the # loss as the original loss name. # tf.summary.scalar(loss_name + ' (raw)', l) # tf.summary.scalar(loss_name, loss_averages.average(l)) # Add dependency to compute loss_averages. with tf.control_dependencies([loss_averages_op]): total_loss = tf.identity(total_loss) # Track the moving averages of all trainable variables. # Note that we maintain a 'double-average' of the BatchNormalization # global statistics. # This is not needed when the number of replicas are small but important # for synchronous distributed training with tens of workers/replicas. exp_moving_averager = tf.train.ExponentialMovingAverage( MOVING_AVERAGE_DECAY, global_step) variables_to_average = ( tf.trainable_variables() + tf.moving_average_variables()) # Add histograms for model variables. # for var in variables_to_average: # tf.summary.histogram(var.op.name, var) # Create synchronous replica optimizer. opt = tf.train.SyncReplicasOptimizer( opt, replicas_to_aggregate=num_replicas_to_aggregate, total_num_replicas=num_workers, variable_averages=exp_moving_averager, variables_to_average=variables_to_average) # batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION) # assert batchnorm_updates, 'Batchnorm updates are missing' # batchnorm_updates_op = tf.group(*batchnorm_updates) # # Add dependency to compute batchnorm_updates. # with tf.control_dependencies([batchnorm_updates_op]): # total_loss = tf.identity(total_loss) # Compute gradients with respect to the loss. # grads = opt.compute_gradients(total_loss) grads0 = opt.compute_gradients(total_loss) grads = [(tf.scalar_mul(tf.cast(batch_size/FLAGS.batch_size, tf.float32), grad), var) for grad, var in grads0] # Add histograms for gradients. # for grad, var in grads: # if grad is not None: # tf.summary.histogram(var.op.name + '/gradients', grad) apply_gradients_op = opt.apply_gradients(grads, global_step=global_step) with tf.control_dependencies([apply_gradients_op]): train_op = tf.identity(total_loss, name='train_op') # Get chief queue_runners and init_tokens, which is used to synchronize # replicas. More details can be found in SyncReplicasOptimizer. chief_queue_runners = [opt.get_chief_queue_runner()] init_tokens_op = opt.get_init_tokens_op() # Create a saver. saver = tf.train.Saver() # Build the summary operation based on the TF collection of Summaries. # summary_op = tf.summary.merge_all() # Build an initialization operation to run below. init_op = tf.global_variables_initializer() # We run the summaries in the same thread as the training operations by # passing in None for summary_op to avoid a summary_thread being started. # Running summaries and training operations in parallel could run out of # GPU memory. sv = tf.train.Supervisor(is_chief=is_chief, logdir=FLAGS.train_dir, init_op=init_op, summary_op=None, global_step=global_step, recovery_wait_secs=1, saver=None, save_model_secs=FLAGS.save_interval_secs) tf.logging.info('%s Supervisor' % datetime.now()) sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=FLAGS.log_device_placement) # Get a session. sess = sv.prepare_or_wait_for_session(target, config=sess_config) # Start the queue runners. queue_runners = tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS) sv.start_queue_runners(sess, queue_runners) tf.logging.info('Started %d queues for processing input data.', len(queue_runners)) if is_chief: sv.start_queue_runners(sess, chief_queue_runners) sess.run(init_tokens_op) # Train, checking for Nans. Concurrently run the summary operation at a # specified interval. Note that the summary_op and train_op never run # simultaneously in order to prevent running out of GPU memory. # next_summary_time = time.time() + FLAGS.save_summaries_secs step = 0 time0 = time.time() batch_size_num = 1 while not sv.should_stop(): try: start_time = time.time() batch_size_num = 32 # batch_size_num = int((int(step)/3*10)) % 100000 + 1 # if step < 5: # batch_size_num = 32 # batch_size_num = (batch_size_num ) % 64 + 1 # else: # batch_size_num = 80 run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() my_images, loss_value, step = sess.run([images, train_op, global_step], feed_dict={batch_size: batch_size_num}, options=run_options, run_metadata=run_metadata) b = time.time() # assert not np.isnan(loss_value), 'Model diverged with loss = NaN' if step > FLAGS.max_steps: break duration = time.time() - start_time thread = threading2.Thread(target=get_computation_time, name="get_computation_time",args=(run_metadata.step_stats,step,)) thread.start() # tl = timeline.Timeline(run_metadata.step_stats) # last_batch_time = tl.get_local_step_duration('sync_token_q_Dequeue') c0 = time.time() # batch_size_num = batchSizeManager.dictate_new_batch_size(FLAGS.task_id, last_batch_time) # batch_size_num = rpcClient.update_batch_size(FLAGS.task_id, last_batch_time, available_cpu, available_memory, step, batch_size_num) # ctf = tl.generate_chrome_trace_format() # with open("timeline.json", 'a') as f: # f.write(ctf) if step % 1 == 0: examples_per_sec = FLAGS.batch_size / float(duration) c = time.time() tf.logging.info("time statistics" + " - train_time: " + str(b-start_time) + " - get_batch_time: " + str(c0-b) + " - get_bs_time: " + str(c-c0) + " - accum_time: " + str(c-time0) + " - batch_size: " + str(batch_size_num)) format_str = ('Worker %d: %s: step %d, loss = %.2f' '(%.1f examples/sec; %.3f sec/batch)') tf.logging.info(format_str % (FLAGS.task_id, datetime.now(), step, loss_value, examples_per_sec, duration)) # Determine if the summary_op should be run on the chief worker. # if is_chief and next_summary_time < time.time(): # tf.logging.info('Running Summary operation on the chief.') # summary_str = sess.run(summary_op) # sv.summary_computed(sess, summary_str) # tf.logging.info('Finished running Summary operation.') # Determine the next time for running the summary. # next_summary_time += FLAGS.save_summaries_secs except: if is_chief: tf.logging.info('Chief got exception while running!') raise # Stop the supervisor. This also waits for service threads to finish. sv.stop()
def apply_gradients(self, grads_and_vars, worker_id, global_step=None, name=None, collect_cdfs=False, # batch_idx_list=None, worker_kill_list=None, num_workers=None, num_batches_per_epoch=None): matrix_to_solve=None, num_batches_per_epoch=None): """Apply gradients to variables. This contains most of the synchronization implementation and also wraps the apply_gradients() from the real optimizer. Args: grads_and_vars: List of (gradient, variable) pairs as returned by compute_gradients(). global_step: Optional Variable to increment by one after the variables have been updated. name: Optional name for the returned operation. Default to the name passed to the Optimizer constructor. Returns: train_op: The op to dequeue a token so the replicas can exit this batch and start the next one. This is executed by each replica. Raises: ValueError: If the grads_and_vars is empty. ValueError: If global step is not provided, the staleness cannot be checked. """ if not grads_and_vars: raise ValueError("Must supply at least one variable") if global_step is None: raise ValueError("Global step is required to check staleness") self._global_step = global_step train_ops = [] aggregated_grad = [] var_list = [] self._local_step = variables.Variable( initial_value=0, trainable=False, collections=[ops.GraphKeys.LOCAL_VARIABLES], dtype=global_step.dtype.base_dtype, name="sync_rep_local_step") self.local_step_init_op = state_ops.assign(self._local_step, global_step._ref()) chief_init_ops = [self.local_step_init_op] self.ready_for_local_init_op = variables.report_uninitialized_variables( variables.all_variables()) # The wait op waits for the current worker to dequeue a token from its respective token queue self._wait_op = self._sync_token_queues[worker_id].dequeue() # Replicas have to wait until they can get a token from the token queue # BEFORE begining to compute gradients. with ops.device(global_step.device): queue_size = self._sync_token_queues[worker_id].size() update_local_step_op = state_ops.assign(self._local_step, global_step._ref()) # Gradient accum creation with ops.name_scope(None, self._name): for grad, var in grads_and_vars: var_list.append(var) tf.logging.info("Grad " + str(grad) + " assigned to " + str(var.device)) with ops.device(var.device): if grad is None: continue elif isinstance(grad, ops.Tensor): grad_accum = data_flow_ops.ConditionalAccumulator( grad.dtype, shape=var.get_shape(), shared_name=var.name + "/grad_accum") else: if not isinstance(grad, ops.IndexedSlices): raise ValueError("Unknown grad type!") grad_accum = data_flow_ops.SparseConditionalAccumulator( grad.dtype, shape=(), shared_name=var.name + "/grad_accum") self._accumulator_list.append((grad_accum, var)) """# Phase 1 gradient computation with ops.control_dependencies([update_local_step_op]): for index, (grad, var) in enumerate(grads_and_vars): with ops.device(var.device): if grad is None: continue elif isinstance(grad, ops.Tensor): grad_accum = self._accumulator_list[index][0] train_ops.append(grad_accum.apply_grad(grad, local_step=self._local_step._ref())) else: if not isinstance(grad, ops.IndexedSlices): raise ValueError("Unknown grad type!") grad_accum = self._accumulator_list[index][0] train_ops.append(grad_accum.apply_indexed_slices_grad( grad, local_step=self._local_step._ref()))""" # Phase 1 gradient computation with ops.control_dependencies([update_local_step_op]): for index, (grad, var) in enumerate(grads_and_vars): print_start_op = logging_ops.Print(global_step, [global_step], message="Starting to apply grads for variable %d" % index) train_ops.append(print_start_op) with ops.device(var.device): ps_step_printer0 = logging_ops.Print(global_step, [global_step], message="global step printer0 on ps") train_ops.append(ps_step_printer0) '''Implement LS computation and solution here''' #b = np.ones(int(num_batches_per_epoch)) b = tf.ones([int(num_batches_per_epoch),1], tf.float32) A = matrix_to_solve # A_for_calc = np.transpose(A) LS_solution = linalg_ops.matrix_solve_ls(A, b, fast=False) LS_calc = tf.reshape(LS_solution, [-1]) weight = tf.slice(LS_calc, [worker_id], [1]) # print_ls_op = logging_ops.Print(LS_calc, [LS_calc], message="Solution for LS!") # train_ops.append(print_ls_op) weighted_grad = tf.scalar_mul(weight[0], grad) '''Kill some workers''' if grad is None: continue elif isinstance(grad, ops.Tensor): grad_accum = self._accumulator_list[index][0] num_accum = grad_accum.num_accumulated() tf.logging.info("Grad Accumed %s, Worker ID: %s" % (str(num_accum), str(worker_id))) with ops.control_dependencies([print_start_op]): with tf.device("job:worker/task:%d" % worker_id): apply_grad_op = grad_accum.apply_grad(grad, # apply_grad_op = grad_accum.apply_grad(weighted_grad, local_step=self._local_step._ref()) with ops.control_dependencies([apply_grad_op]): finished_print_op = logging_ops.Print(global_step, [global_step], message="Done applying grads for variable %d" % index) train_ops.append(finished_print_op) else: if not isinstance(grad, ops.IndexedSlices): raise ValueError("Unknown grad type!") grad_accum = self._accumulator_list[index][0] with ops.control_dependencies([print_start_op]): with tf.device("job:worker/task:%d" % worker_id): apply_grad_op = grad_accum.apply_indexed_slices_grad( grad, local_step=self._local_step._ref()) # weighted_grad, local_step=self._local_step._ref()) with ops.control_dependencies([apply_grad_op]): finished_print_op = logging_ops.Print(global_step, [global_step], message="Done applying grads for variable %d" % index) train_ops.append(finished_print_op) # Phase 2 gradient applying for index, (grad, var) in enumerate(grads_and_vars): with ops.device(var.device): work_idx_print1 = logging_ops.Print(worker_id, [worker_id], message="worker id for aggregate grad") ps_step_printer1 = logging_ops.Print(global_step, [global_step], message="global step printer1 on ps") num_replica_aggragate = logging_ops.Print(self._replicas_to_aggregate, [self._replicas_to_aggregate], message="num replica aggregate") train_ops.append(work_idx_print1) train_ops.append(ps_step_printer1) train_ops.append(num_replica_aggragate) grad_accum = self._accumulator_list[index][0] if grad is None: aggregated_grad.append(None) elif isinstance(grad, ops.Tensor): if collect_cdfs: # aggregated_grad.append(grad_accum.take_grad(self._total_num_replicas)) aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate)) else: aggregated_grad.append(grad_accum.take_grad(1)) else: if collect_cdfs: # aggregated_grad.append(grad_accum.take_grad(self._total_num_replicas)) aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate)) else: aggregated_grad.append(grad_accum.take_indexed_slices_grad(1)) aggregated_grads_and_vars = zip(aggregated_grad, var_list) # Some debug operations self.print_sizes = logging_ops.Print(global_step, [self._sync_token_queues[i].size() for i in range(self._total_num_replicas)], message="queue sizes") self.print_accum_sizes = logging_ops.Print(self._local_step, [x[0].num_accumulated() for x in self._accumulator_list] + [worker_id], message="Accum sizes") self.print_local_step = logging_ops.Print(self._local_step, [self._local_step._ref(), global_step._ref()], message="local vs global step") # sync_op will be assigned to the same device as the global step. with ops.device(global_step.device), ops.name_scope(""): with ops.control_dependencies([self.print_accum_sizes]): update_op = self._opt.apply_gradients(aggregated_grads_and_vars, global_step) self._update_op = update_op with ops.control_dependencies([update_op]): sync_op = [] for cur_worker_id in range(self._total_num_replicas): sync_op.append(self._sync_token_queues[cur_worker_id].enqueue(global_step)) sync_op = control_flow_ops.group(*(sync_op)) # dummy_queue is passed to the queue runner. Don't use the real queues # because the queue runner doesn't automatically reopen it once it # closed queues in PS devices. dummy_queue = ( data_flow_ops.FIFOQueue(1, types_pb2.DT_INT32, shapes=(), shared_name="dummy_queue")) self._chief_queue_runner = queue_runner.QueueRunner(dummy_queue, [sync_op]) with ops.device(global_step.device), ops.name_scope(""): with ops.control_dependencies(train_ops): # Worker finished applying gradients. Add token to phase1_finished_queue train_op = logging_ops.Print(self._local_step._ref(), [x[0].num_accumulated() for x in self._accumulator_list] + [worker_id], message="Finished worker updates", name="FinishedWorkerUpdatesPrint") for accum, var in self._accumulator_list: with ops.device(var.device): chief_init_ops.append( accum.set_global_step( global_step, name="SetGlobalStep")) self.chief_init_op = control_flow_ops.group(*(chief_init_ops)) self._gradients_applied = True return train_op
def main(): """Create the model and start the training.""" args = get_arguments() args.snapshot_dir=args.snapshot_dir.replace('DeepDICD/','DeepDICD/'+args.model_name+'-'+args.domain+'-') print(toMagenta(args.snapshot_dir)) ss=args.domain.split('-') if ss[0]=='D': args.list1=args.list1.replace('amazon.txt','dslr.txt') elif ss[0]=='W': args.list1=args.list1.replace('amazon.txt','webcam.txt') if ss[1]=='A': args.list2=args.list2.replace('dslr.txt','amazon.txt') elif ss[1]=='W': args.list2=args.list2.replace('dslr.txt','webcam.txt') print(toMagenta(args.list1)) print(toMagenta(args.list2)) start_steps=args.start_steps h=args.h w=args.w # construct data generator file1 = open(args.list1) num1 = len(file1.readlines()) file2 = open(args.list2) num2 = len(file2.readlines()) file1.close() file2.close() steps_per_epoch=int((num1/(args.batch_size))) num_steps=int(steps_per_epoch*args.num_epochs) val_num_steps=int(num2/args.batch_size) print(toCyan('src domain: {:d}, tar domain {:d}'.format(num1,num2))) print(toCyan('steps_per_epoch x num_epochs:{:d} x {:d}'.format(steps_per_epoch,args.num_epochs))) # Chong # split_batch_size=int(args.batch_size/hvd.size()) myDataloader=Dataloader(args.img_dir,args.list1,args.list2,args.batch_size,args.h,args.w,args.num_threads) src_img=myDataloader.simg_batch src_label=myDataloader.slabel_batch tar_img=myDataloader.timg_batch tar_label=myDataloader.tlabel_batch coord = tf.train.Coordinator() # Learning Startegy Settings baseLR1 = tf.constant(args.lr1) baseLR2 = tf.constant(args.lr2) step_ph = tf.placeholder(dtype=tf.float32, shape=()) if args.learning_strategy==0: lr1=baseLR1/tf.pow(1+0.001*step_ph/steps_per_epoch,0.75) lr2=baseLR2/tf.pow(1+0.001*step_ph/steps_per_epoch,0.75) elif args.learning_strategy==1: lr1=baseLR1/tf.pow(1+0.0005*step_ph/steps_per_epoch,0.75) lr2=baseLR2/tf.pow(1+0.0005*step_ph/steps_per_epoch,0.75) if args.learning_strategy==2: lr1=baseLR1/tf.pow(1+0.005*step_ph/steps_per_epoch,0.75) lr2=baseLR2/tf.pow(1+0.005*step_ph/steps_per_epoch,0.75) if args.learning_strategy==3: lr1=baseLR1/tf.pow(1+0.001*step_ph,0.75) lr2=baseLR2/tf.pow(1+0.001*step_ph,0.75) # lr1 = tf.scalar_mul(baseLR1, tf.pow((1 - step_ph / num_steps), args.power)) # lr2 = tf.scalar_mul(baseLR2, tf.pow((1 - step_ph / num_steps), args.power)) # lr1=baseLR1/tf.pow(1+0.001*step_ph,0.75) # lr2=baseLR2/tf.pow(1+0.001*step_ph,0.75) # Polynomial_decay # lr1=baseLR1 # lr2=baseLR2 # decay_steps=steps_per_epoch*10 # lr1=tf.train.exponential_decay(baseLR1,step_ph,decay_steps,0.1,staircase=True) # lr2=tf.train.exponential_decay(baseLR2,step_ph,decay_steps,0.1,staircase=True) keep_prob=tf.placeholder(dtype=tf.float32,shape=()) p=(1- tf.scalar_mul(1., tf.pow((1 - step_ph / num_steps), args.power))) alpha1 = adaptation_factor(step_ph*1.0/10000) # alpha1 = tf.constant(1.,tf.float32) alpha2 = 0.01*p # alpha2 = 0 model = DeepDICDModel(args,keep_prob,src_img,src_label,tar_img,tar_label) model.build_losses() model.build_outputs() all_trainable_var=[v for v in tf.trainable_variables()] fine_tune_var = [v for v in all_trainable_var if 'fc8' not in v.name and 'fc9' not in v.name and 'domain' not in v.name] fine_tune_var_weights=[v for v in fine_tune_var if 'weights' in v.name] fine_tune_var_bias=[v for v in fine_tune_var if 'bias' in v.name] retrain_var = [v for v in all_trainable_var if 'fc8' in v.name or 'fc9' in v.name] retrain_var_weights=[v for v in retrain_var if 'weights' in v.name] retrain_var_bias=[v for v in retrain_var if 'bias' in v.name] domain_var=[v for v in all_trainable_var if 'domain' in v.name] domain_var_weights=[v for v in domain_var if 'weights' in v.name] domain_var_bias=[v for v in domain_var if 'bias' in v.name] print(toGreen(domain_var_bias)) model.build_regular_losses(domain_var_weights,fine_tune_var_weights+fine_tune_var_weights) summary_=model.build_summary() # ------------- Loss Function ------------------ # F_loss=model.classify_loss_src+model.dregular_loss+model.G_loss \ # + model.class_wise_adaptation_loss \ # + alpha2*(model.sintra_loss+model.tintra_loss-0.1*model.sinter_loss-0.1*model.tinter_loss) # F_loss=model.classify_loss_src+model.dregular_loss+alpha1*model.G_loss \ # + alpha1*model.class_wise_adaptation_loss \ # + alpha2*(model.sintra_loss+model.tintra_loss-0.1*model.sinter_loss-0.1*model.tinter_loss) F_loss=model.classify_loss_src+model.gregular_loss+alpha1*model.G_loss \ + alpha1*model.class_wise_adaptation_loss \ + alpha2*(model.sintra_loss+model.tintra_loss) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.no_update_mean_var == True: update_ops = None else: print(toMagenta('updating mean and var in batchnorm')) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_1_1 = tf.train.MomentumOptimizer(lr1, args.momentum) grads_1_1 = tf.gradients(F_loss, fine_tune_var_weights) train_op_1_1 = opt_1_1.apply_gradients(zip(grads_1_1, fine_tune_var_weights)) opt_1_2 = tf.train.MomentumOptimizer(2*lr1, args.momentum) grads_1_2 = tf.gradients(F_loss, fine_tune_var_bias) train_op_1_2 = opt_1_2.apply_gradients(zip(grads_1_2, fine_tune_var_bias)) opt_2_1 = tf.train.MomentumOptimizer(lr2, args.momentum) grads_2_1 = tf.gradients(F_loss, retrain_var_weights) train_op_2_1 = opt_2_1.apply_gradients(zip(grads_2_1, retrain_var_weights)) opt_2_2 = tf.train.MomentumOptimizer(2*lr2, args.momentum) grads_2_2 = tf.gradients(F_loss, retrain_var_bias) train_op_2_2 = opt_2_2.apply_gradients(zip(grads_2_2, retrain_var_bias)) opt_3_1 = tf.train.MomentumOptimizer(lr2,args.momentum) grads_3_1 = tf.gradients(model.dregular_loss+model.D_loss, domain_var_weights) train_op_3_1 = opt_3_1.apply_gradients(zip(grads_3_1, domain_var_weights)) opt_3_2 = tf.train.MomentumOptimizer(2*lr2,args.momentum) grads_3_2 = tf.gradients(model.dregular_loss+model.D_loss, domain_var_bias) train_op_3_2 = opt_3_2.apply_gradients(zip(grads_3_2, domain_var_bias)) train_op = tf.group(train_op_1_1,train_op_1_2,train_op_2_1,train_op_2_2,train_op_3_1,train_op_3_2) # Set up tf session and initialize variables. # config = tf.ConfigProto()#Chong config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.4 sess = tf.Session(config=config) init_local=tf.local_variables_initializer() init = tf.global_variables_initializer() # construct summary summary_.append(tf.summary.scalar( 'train/lr1', lr1)) summary_.append(tf.summary.scalar( 'train/lr2', lr2)) summary_.append(tf.summary.scalar( 'train/alpha1', alpha1)) summary_.append(tf.summary.scalar( 'train/alpha2', alpha2)) summary_merged=tf.summary.merge(summary_) FinalSummary = tf.summary.FileWriter(args.snapshot_dir,sess.graph) # init sess.run([init_local,init]) # Saver for storing checkpoints of the model. var=tf.global_variables() skip_var=['fc8','fc9'] saver = tf.train.Saver(var_list=var, max_to_keep=5) ckpt = tf.train.get_checkpoint_state(args.resume_from) if ckpt and ckpt.model_checkpoint_path and args.resume: loader = tf.train.Saver(var_list=var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) elif not args.not_load_pretrained: print(toRed('Restore from pre-trained model...' + args.restore_from)) model.load_initial_weights(sess, args.restore_from, skip_var) #Chong:0531 # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. acc2_history=0 for step in range(start_steps,num_steps): start_time = time.time() feed_dict = {step_ph: step, keep_prob: 0.5} summary, total_loss, _= sess.run([summary_merged,F_loss,train_op], feed_dict=feed_dict) FinalSummary.add_summary(summary, step) duration = time.time() - start_time remain_time=duration*(num_steps-step)/3600 print('\r',toCyan('{:s}:{:d}-{:d}-{:d} total loss = {:.3f},({:.3f} sec/step, ERT: {:.3f})'.format(args.model_name+'-'+args.domain,step%steps_per_epoch, step//steps_per_epoch,args.num_epochs,total_loss, duration,remain_time)),end='') if step % args.test_every == 0: acc1,acc2=0,0 for jj in range(val_num_steps): feed_dict = {keep_prob: 1} src_acc,tar_acc= sess.run([model.src_acc,model.tar_acc], feed_dict=feed_dict) acc1+=np.sum(src_acc) acc2+=np.sum(tar_acc) acc1=acc1/(val_num_steps*args.batch_size) acc2=acc2/(val_num_steps*args.batch_size) # pdb.set_trace() test_summary = tf.Summary() test_summary.value.add(tag='test/source_accuracy',simple_value= acc1) test_summary.value.add(tag='test/target_accuracy',simple_value= acc2) FinalSummary.add_summary(test_summary, step) if acc2>acc2_history: save(saver, sess, args.snapshot_dir, step) acc2_history=acc2 coord.request_stop() coord.join(threads) sess.close()
def main(): # Create model and start training h, w = INPUT_SIZE X = tf.placeholder(tf.float32, shape=[None, h, w, 3], name='X') Y = tf.placeholder(tf.uint8, shape=[None, h, w, 1], name='Y') is_training = tf.placeholder(tf.bool, name='is_training') net = DeepLabResNetModel(X, is_training, NUM_CLASSES, ATROUS_BLOCKS) raw_output = net.output # Trainable Variables # restore_vars = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last] all_trainable = [ v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name ] fc_trainable = [v for v in all_trainable if 'fc' in v.name] conv_trainable = [v for v in all_trainable if 'fc' not in v.name] fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # Predictions: ignoring all predictions with labels greater or equal than n_classes raw_prediction = tf.reshape(raw_output, [-1, NUM_CLASSES]) label_proc = prepare_labels(Y, tf.stack(raw_output.get_shape()[1:3]), num_classes=NUM_CLASSES, one_hot=False) raw_gt = tf.reshape(label_proc, [ -1, ]) indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, NUM_CLASSES - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) prediction = tf.gather(raw_prediction, indices) # Pixel-wise Softmax Loss loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) l2_losses = [ WEIGHT_DECAY * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name ] reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses) variable_summaries(reduced_loss, name='loss') variable_summaries(loss, name='loss_origin') # Processed predictions: for visualization raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(X)[1:3, ]) raw_output_up = tf.argmax(raw_output_up, dimension=3) pred = tf.expand_dims(raw_output_up, dim=3) # Define loss and optimization parameters base_lr = tf.constant(LEARNING_RATE, tf.float64) global_step = tf.Variable(0, trainable=False, name='global_step') increment_step = tf.assign(global_step, global_step + 1) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - global_step / NUM_STEPS), POWER)) learning_rate = tf.maximum(learning_rate, 8e-7) opt_conv = tf.train.MomentumOptimizer(learning_rate, MOMENTUM) opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 5.0, MOMENTUM) opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 10.0, MOMENTUM) grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable) grads_conv = grads[:len(conv_trainable)] grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) + len(fc_w_trainable))] grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):] train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable)) train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable)) train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable)) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = tf.group(increment_step, train_op_conv, train_op_fc_w, train_op_fc_b) # initial_learning_rate = 1e-2 # learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step, 300, 0.96) # adam = tf.train.AdamOptimizer(learning_rate).minimize(reduced_loss, global_step=global_step) # Image Summary images_summary = tf.py_func(inv_preprocess, [X, SAVE_NUM_IMAGES, IMG_MEAN], tf.uint8) preds_summary = tf.py_func(decode_labels, [pred, SAVE_NUM_IMAGES, NUM_CLASSES], tf.uint8) labels_summary = tf.py_func(decode_labels, [Y, SAVE_NUM_IMAGES, NUM_CLASSES], tf.uint8) image_summaries = [images_summary, preds_summary, labels_summary] image_summary = tf.summary.image('images', tf.concat(axis=2, values=image_summaries), max_outputs=SAVE_NUM_IMAGES) # Variable Summary variable_summaries(fc_w_trainable, 'fc_w') variable_summaries(fc_b_trainable, 'fc_b') variable_summaries(learning_rate, 'learning_rate') # variable_summaries(net.weights, 'aconv_w') # variable_summaries(net.biases, 'aconv_b') total_summary = tf.summary.merge_all() tb_train_dir = os.path.join(SNAPSHOT_DIR, 'train') tb_val_dir = os.path.join(SNAPSHOT_DIR, 'verify') summary_writer = tf.summary.FileWriter(tb_train_dir, graph=tf.get_default_graph()) verify_writer = tf.summary.FileWriter(tb_val_dir) train_data, val_data = read_data() train_data_count = len(train_data) batch_size = BATCH_SIZE print('batch size: %s, total step: %s, train data num: %s' % (batch_size, NUM_STEPS, train_data_count)) # Set up session with tf.Session() as sess: tf.global_variables_initializer().run() saver = tf.train.Saver(max_to_keep=3) if SNAPSHOT_DIR is not None and os.path.exists(SNAPSHOT_DIR): loader = tf.train.Saver() load_model(loader, sess, SNAPSHOT_DIR) # export pb if False: import export_pb export_pb.save_graph_with_weight( sess, ['ExpandDims'], '/data/pb/traffic_line_deeplab_resnet_cut05_271_loss_0_0_93.pb' ) start_index = 0 for step in range(NUM_STEPS): # start_time = time.time() if start_index + batch_size > train_data_count: start_index = 0 random.shuffle(train_data) _train_data = train_data[start_index:start_index + batch_size] start_index += batch_size after_processing_data = [] start_time = time.time() for x_img, y_data in _train_data: # tmp_img = augmentation(x_img) # x_data = np.asarray(tmp_img).astype('float32') / 255.0 after_processing_data.append((x_img, y_data)) vec = [d[0] for d in after_processing_data] vec2 = [d[1] for d in after_processing_data] # data_process_time = time.time() - start_time if step % SAVE_SUMMARY_EVERY == 0: feed = [ reduced_loss, pred, total_summary, global_step, train_op ] loss_value, preds, summary, total_steps, _ = \ sess.run(feed, feed_dict={X: vec, Y: vec2, is_training: True}) summary_writer.add_summary(summary, total_steps) else: feed = [reduced_loss, global_step, train_op] loss_value, total_steps, _ = sess.run(feed, feed_dict={ X: vec, Y: vec2, is_training: True }) if step % SAVE_MODEL_EVERY == 0: save_model(saver, sess, SNAPSHOT_DIR, global_step) duration = time.time() - start_time results = 'global step: {:d}, step: {:d} \t loss = {:.3f}, ({:.3f} secs)'\ .format(total_steps, step, loss_value, duration) if step % WRITE_EVERY == 0: with open(WRITE_FILE, 'a') as f: f.write(results + '\n') print(results) if step % VAL_EVERY == 0: random.shuffle(val_data) vec = [d[0] for d in val_data[:batch_size]] vec2 = [d[1] for d in val_data[:batch_size]] feed = [reduced_loss, pred, total_summary, global_step] loss_value, preds, summary, total_steps = \ sess.run(feed, feed_dict={X: vec, Y: vec2, is_training: False}) verify_writer.add_summary(summary, total_steps) print('[Verify]', step, loss_value)
def __init__(self, num_classes, vocab_size, shape_domain_size, char_domain_size, char_size, embedding_size, shape_size, nonlinearity, viterbi, hidden_dim, char_embeddings, embeddings=None): self.num_classes = num_classes self.shape_domain_size = shape_domain_size self.char_domain_size = char_domain_size self.char_size = char_size self.embedding_size = embedding_size self.shape_size = shape_size self.hidden_dim = hidden_dim self.nonlinearity = nonlinearity self.char_embeddings = char_embeddings self.viterbi = viterbi # word embedding input self.input_x1 = tf.placeholder(tf.int64, [None, None], name="input_x1") # shape embedding input self.input_x2 = tf.placeholder(tf.int64, [None, None], name="input_x2") # labels self.input_y = tf.placeholder(tf.int64, [None, None], name="input_y") # padding mask self.input_mask = tf.placeholder(tf.float32, [None, None], name="input_mask") self.batch_size = tf.placeholder(tf.int32, None, name="batch_size") self.max_seq_len = tf.placeholder(tf.int32, None, name="max_seq_len") # sequence lengths self.sequence_lengths = tf.placeholder(tf.int32, [None, None], name="sequence_lengths") # dropout and l2 penalties self.middle_dropout_keep_prob = tf.placeholder_with_default( 1.0, [], name="middle_dropout_keep_prob") self.hidden_dropout_keep_prob = tf.placeholder_with_default( 1.0, [], name="hidden_dropout_keep_prob") self.input_dropout_keep_prob = tf.placeholder_with_default( 1.0, [], name="input_dropout_keep_prob") self.word_dropout_keep_prob = tf.placeholder_with_default( 1.0, [], name="word_dropout_keep_prob") self.l2_penalty = tf.placeholder_with_default(0.0, [], name="l2_penalty") self.projection = tf.placeholder_with_default(False, [], name="projection") self.drop_penalty = tf.placeholder_with_default(0.0, [], name="drop_penalty") # Keeping track of l2 regularization loss (optional) self.l2_loss = tf.constant(0.0) # set the pad token to a constant 0 vector self.word_zero_pad = tf.constant(0.0, dtype=tf.float32, shape=[1, embedding_size]) self.shape_zero_pad = tf.constant(0.0, dtype=tf.float32, shape=[1, shape_size]) self.char_zero_pad = tf.constant(0.0, dtype=tf.float32, shape=[1, char_size]) self.use_characters = char_size != 0 self.use_shape = shape_size != 0 if self.viterbi: self.transition_params = tf.get_variable( "transitions", [num_classes, num_classes]) # Embedding layer # with tf.device('/cpu:0'), tf.name_scope("embedding"): word_embeddings_shape = (vocab_size - 1, embedding_size) self.w_e = tf_utils.initialize_embeddings(word_embeddings_shape, name="w_e", pretrained=embeddings) nonzero_elements = tf.not_equal(self.sequence_lengths, tf.zeros_like(self.sequence_lengths)) count_nonzero_per_row = tf.reduce_sum(tf.to_int32(nonzero_elements), axis=1) self.flat_sequence_lengths = tf.add( tf.reduce_sum(self.sequence_lengths, 1), tf.scalar_mul(2, count_nonzero_per_row)) self.unflat_scores, self.hidden_layer = self.forward( self.input_x1, self.input_x2, self.max_seq_len, self.hidden_dropout_keep_prob, self.input_dropout_keep_prob, self.middle_dropout_keep_prob, reuse=False) # Calculate mean cross-entropy loss with tf.name_scope("loss"): labels = tf.cast(self.input_y, 'int32') if viterbi: log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood( self.unflat_scores, labels, self.flat_sequence_lengths, transition_params=self.transition_params) # self.transition_params = transition_params self.loss = tf.reduce_mean(-log_likelihood) else: losses = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.unflat_scores, labels=labels) masked_losses = tf.multiply(losses, self.input_mask) self.loss = tf.div(tf.reduce_sum(masked_losses), tf.reduce_sum(self.input_mask)) self.loss += self.l2_penalty * self.l2_loss self.unflat_no_dropout_scores, _ = self.forward( self.input_x1, self.input_x2, self.max_seq_len, 1.0, 1.0, 1.0) drop_loss = tf.nn.l2_loss( tf.subtract(self.unflat_scores, self.unflat_no_dropout_scores)) self.loss += self.drop_penalty * drop_loss # Accuracy with tf.name_scope("predictions"): if viterbi: self.predictions = self.unflat_scores else: self.predictions = tf.argmax(self.unflat_scores, 2)
def train(): learning_rate = args.lr model_path = args.model total_epoch = args.epoch teacher = nin() student = lenet() if args.noisy == True: drop_scale = 1 / args.Nratio noisy_mask = tf.nn.dropout(tf.constant( np.float32(np.ones((batch_size, 1))) / drop_scale), keep_prob=args.Nratio) #(batchsize,1) gaussian = tf.random_normal(shape=[batch_size, 1], mean=0.0, stddev=args.Nsigma) noisy = tf.mul(noisy_mask, gaussian) #noisy_add = tf.add(tf.constant(np.float32(np.ones((batch_size,1)))), noisy) teacher = tf.mul(teacher, tf.tile(noisy, tf.constant([1, 10]))) #(batchsize,10) #teacher = tf.add(teacher, tf.tile(noisy,tf.constant([1,10]))) print(bcolors.G + "prepare for training, noisy mode" + bcolors.END) tf_loss = tf.nn.l2_loss(teacher - student) / batch_size elif args.KD == True: # correct Hinton method at 2017.1.3 print(bcolors.G + "prepare for training, knowledge distilling mode" + bcolors.END) one_hot = tf.one_hot(y, n_classes, 1.0, 0.0) #one_hot = tf.cast(one_hot_int, tf.float32) teacher_tau = tf.scalar_mul(1.0 / args.tau, teacher) student_tau = tf.scalar_mul(1.0 / args.tau, student) objective1 = tf.nn.sigmoid_cross_entropy_with_logits( student_tau, one_hot) objective2 = tf.scalar_mul(0.5, tf.square(student_tau - teacher_tau)) tf_loss = (args.lamda * tf.reduce_sum(objective1) + (1 - args.lamda) * tf.reduce_sum(objective2)) / batch_size else: print(bcolors.G + "prepare for training, NIPS2014 mode" + bcolors.END) tf_loss = tf.nn.l2_loss(teacher - student) / batch_size optimizer1 = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(tf_loss) optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate / 10).minimize(tf_loss) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5) sess = tf.InteractiveSession(config=tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True)) tf.initialize_all_variables().run() with tf.device('/cpu:0'): saver = tf.train.Saver(max_to_keep=100) #saver.restore(sess, os.path.join(model_path,'model-99') data, label = read_cifar10('train') index = np.array(range(len(data))) # index randomly ordered mean = cal_mean() begin = time.time() iterations = len(data) / batch_size decay_step = int(total_epoch * 0.8) cnt = 0 dropout_rate = args.dropout print(bcolors.G + "number of iterations (per epoch) =" + str(len(data) / batch_size) + bcolors.END) for i in range(total_epoch): np.random.shuffle(index) cost_sum = 0 for j in range(iterations): batch_x = np.float32( data[index[j * batch_size:(j + 1) * batch_size]]) - mean batch_y = np.squeeze( np.float32(label[index[j * batch_size:(j + 1) * batch_size]])) if cnt / decay_step == 0: lr = learning_rate _, cost = sess.run([optimizer1, tf_loss], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1 - dropout_rate }) elif cnt / decay_step == 1: lr = learning_rate / 10 _, cost = sess.run([optimizer2, tf_loss], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1 - dropout_rate }) cost_sum += cost #pdb.set_trace() #if (j % int(iterations*0.25) == 0): # print ("epoch %d-iter %d, cost = %f , avg-cost = %f"%(i, j, cost, cost/n_classes)) # sys.stdout.flush() cnt += 1 avg_time = time.time() - begin print( "epoch %d - avg. %f seconds in each epoch, lr = %.0e, cost = %f , avg-cost-per-logits = %f" % (i, avg_time / cnt, lr, cost_sum, cost_sum / iterations / n_classes)) if np.mod(i + 1, 10) == 0: print("Epoch ", i + 1, " is done. Saving the model ...") with tf.device('/cpu:0'): if not os.path.exists(model_path): os.makedirs(model_path) saver.save(sess, os.path.join(model_path, 'model'), global_step=i) sys.stdout.flush()
def loss(self, predictions_dict, groundtruth_dict): """Define loss functions given the prediction and the groundtruth dictionary of tensors Args: predictions_dict: a dictionary holding predicted tensors groundtruth_dict: a dictionary holding groundtruth tensors for computing the loss Returns: loss_dict: a dictionary mapping strings (loss names) to scalar tensors representing loss values. """ loss_config = self.model_config.losses batch_size = self.model_config.batch_size batch_reduce_factor = 2 if self.model_config.input_queue_type in ['source_only', 'target_only']: batch_reduce_factor = 1 loss_dict = { 'triplet_loss': None, 'domain_loss': None, 'classify_loss': None, 'reconstruction_loss': None, 'wasserstein_loss': None, 'kl_loss': None } if loss_config.reconstruction_loss.use_reconstruction_loss: reconstruction_loss = losses.reconstruction_loss(inputs=predictions_dict[PREPROCESSED_INPUTS_KEY], outputs=predictions_dict[PRED_RECONSTRUCTION_KEY], scale=float(loss_config.reconstruction_loss.reconstruction_loss_weight)) loss_dict['reconstruction_loss'] = reconstruction_loss if loss_config.triplet_loss.use_triplet_loss: triplet_loss = losses.triplet_loss(labels=tf.argmax(groundtruth_dict[mnist_dataset_builder.CLASS_LABEL_KEY], axis=1), embeddings=predictions_dict[EMBEDDING_FEATURES_KEY], margin=float(loss_config.triplet_loss.triplet_loss_margin), scale=float(loss_config.triplet_loss.triplet_loss_weight), batch_reduce_factor=batch_reduce_factor) loss_dict['triplet_loss'] = triplet_loss if loss_config.domain_loss.use_domain_loss: domain_loss, domain_accuracy, _ = losses.softmax_cross_entropy_with_logits_v2( logits=predictions_dict[PRED_DOMAIN_KEY], labels=groundtruth_dict[ mnist_dataset_builder.DOMAIN_LABEL_KEY], scale=float( loss_config.domain_loss.domain_loss_weight), loss_name='Loss/domain_loss') loss_dict['domain_loss'] = domain_loss # TODO(): Add classification_accuracy to the tensorboard summary if loss_config.source_classification_loss.use_source_classification_loss: source_classify_loss, classify_source_accuracy, classify_target_accuracy = losses.softmax_cross_entropy_with_logits_v2( logits=predictions_dict[PRED_CLASSES_KEY], labels=groundtruth_dict[mnist_dataset_builder.CLASS_LABEL_KEY], scale=float(loss_config.source_classification_loss.source_classification_loss_weight), batch_reduce_factor=batch_reduce_factor, loss_name='Loss/source_classify_loss') loss_dict['classify_loss'] = source_classify_loss # TODO(): Recheck the output of the wasserstein loss, adjust the batch size if loss_config.wasserstein_loss.use_wasserstein_loss: wasser_loss = -tf.contrib.gan.losses.wargs.wasserstein_discriminator_loss( discriminator_real_outputs=predictions_dict[PRED_DOMAIN_KEY][:batch_size // batch_reduce_factor], discriminator_gen_outputs=predictions_dict[PRED_DOMAIN_KEY][batch_size // batch_reduce_factor:]) wasser_loss = tf.scalar_mul(loss_config.wasserstein_loss.wasserstein_loss_weight, wasser_loss) loss_dict['wasserstein_loss'] = wasser_loss if loss_config.kl_divergence_loss.use_kl_divergence_loss: kl_loss = losses.kl_div(predictions_dict[EMBEDDING_FEATURES_KEY][:batch_size // batch_reduce_factor], predictions_dict[EMBEDDING_FEATURES_KEY][batch_size // batch_reduce_factor:]) kl_loss = tf.scalar_mul(loss_config.use_kl_divergence_loss.kl_divergence_loss_weight, kl_loss) loss_dict['kl_loss'] = kl_loss # loss = triplet_loss + domain_loss + reconstruction_loss + classify_loss # # loss_dict = { # 'triplet_loss': triplet_loss, # 'domain_loss': domain_loss, # 'classify_loss': source_classify_loss, # 'reconstruction_loss': reconstruction_loss, # 'wasserstein_loss': wasser_loss, # 'kl_loss': kl_loss # } return loss_dict
def build_network(d): # Define hyperparameters d = d learning_rate = 2e-5 l2norm_scaling = 1e-10 global_norm_gradient_clipping_ratio = 0.65 # Define a placeholder for the answers to the decision problems route_exists = tf.placeholder( tf.float32, shape = (None,), name = 'route_exists' ) # Define a placeholder for the cost of each route route_costs = tf.placeholder( tf.float32, shape=(None,1), name='route_costs') # Define a placeholder for the edges mask edges_mask = tf.placeholder( tf.float32, shape = (None,), name = 'edges_mask' ) # Define placeholders for the list of number of vertices and edges per instance n_vertices = tf.placeholder( tf.int32, shape = (None,), name = 'n_vertices') n_edges = tf.placeholder( tf.int32, shape = (None,), name = 'edges') # EV_matrix = tf.placeholder( tf.float32, shape = (None,None), name = "EV" ) edge_weight = tf.placeholder( tf.float32, shape = (None,1), name = "edge_weight" ) target_cost = tf.placeholder( tf.float32, shape = (None,1), name = "target_cost" ) time_steps = tf.placeholder( tf.int32, shape = (), name = "time_steps" ) initial_embedding_mlp = Mlp( layer_sizes = [ d for _ in range(3) ], activations = [ tf.nn.relu for _ in range(3) ], output_size = d, name = 'E_init_MLP', name_internal_layers = True, kernel_initializer = tf.contrib.layers.xavier_initializer(), bias_initializer = tf.zeros_initializer() ) edge_initial_embeddings = initial_embedding_mlp( tf.concat( [ edge_weight, target_cost ], axis = 1 ) ) vertex_initial_embeddings = tf.get_variable( initializer = tf.random_normal( (1,d) ), dtype = tf.float32, name='V_init' ) total_n = tf.shape( EV_matrix )[1] tiled_and_normalized_vertex_initial_embeddings = tf.tile( tf.div( vertex_initial_embeddings, tf.sqrt( tf.cast( total_n, tf.float32 ) ) ), [ total_n, 1 ] ) # Define GNN dictionary GNN = {} # Define Graph neural network gnn = GraphNN( { # V is the set of vertex embeddings 'V': d, # E is the set of edge embeddings 'E': d }, { # M is a E×V adjacency matrix connecting each edge to the vertices it is connected to 'EV': ('E','V') }, { # V_msg_E is a MLP which computes messages from vertex embeddings to edge embeddings 'V_msg_E': ('V','E'), # E_msg_V is a MLP which computes messages from edge embeddings to vertex embeddings 'E_msg_V': ('E','V') }, { # V(t+1) ← Vu( EVᵀ × E_msg_V(E(t)) ) 'V': [ { 'mat': 'EV', 'msg': 'E_msg_V', 'transpose?': True, 'var': 'E' } ], # E(t+1) ← Eu( EV × V_msg_E(V(t)), W, C ) 'E': [ { 'mat': 'EV', 'msg': 'V_msg_E', 'var': 'V' } ] }, name='TSP' ) # Populate GNN dictionary GNN['gnn'] = gnn GNN['route_exists'] = route_exists GNN['route_costs'] = route_costs GNN['edges_mask'] = edges_mask GNN['n_vertices'] = n_vertices GNN['n_edges'] = n_edges GNN["EV"] = EV_matrix GNN["W"] = edge_weight GNN["C"] = target_cost GNN["time_steps"] = time_steps # Define E_vote, which will compute one logit for each edge E_vote_MLP = Mlp( layer_sizes = [ d for _ in range(3) ], activations = [ tf.nn.relu for _ in range(3) ], output_size = 1, name = 'E_vote', name_internal_layers = True, kernel_initializer = tf.contrib.layers.xavier_initializer(), bias_initializer = tf.zeros_initializer() ) vote_bias = tf.get_variable(initializer=tf.zeros_initializer(), shape=(), dtype=tf.float32, name='vote_bias') # Get the last embeddings last_states = gnn( { "EV": EV_matrix }, { "V": tiled_and_normalized_vertex_initial_embeddings, "E": edge_initial_embeddings }, time_steps = time_steps ) GNN["last_states"] = last_states E_n = last_states['E'].h # Compute a vote for each embedding #E_vote = tf.reshape(E_vote_MLP(tf.concat([E_n,route_costs], axis=1)), [-1]) E_vote = tf.reshape(E_vote_MLP(E_n), [-1]) E_prob = tf.sigmoid(E_vote) # Compute the number of problems in the batch num_problems = tf.shape(n_vertices)[0] # n_edges_acc[i] is the number of edges in the batch up to the i-th instance n_edges_acc = tf.map_fn(lambda i: tf.reduce_sum(tf.gather(n_edges, tf.range(0,i))), tf.range(0,num_problems)) # Compute decision predictions (one per problem) _, pred_logits = tf.while_loop( lambda i, predictions: tf.less(i, num_problems), lambda i, predictions: ( (i+1), predictions.write( i, #tf.reduce_mean(tf.gather(E_vote, tf.range(n_edges_acc[i], n_edges_acc[i] + n_edges[i]))) tf.reduce_mean( E_vote[n_edges_acc[i]:n_edges_acc[i]+n_edges[i]] ) ) ), [0, tf.TensorArray(size=num_problems, dtype=tf.float32)] ) pred_logits = pred_logits.stack() + vote_bias GNN['predictions'] = tf.sigmoid(pred_logits) # Count the number of edges that appear in the solution pos_edges_n = tf.reduce_sum(edges_mask) # Count the number of edges that do not appear in the solution neg_edges_n = tf.reduce_sum(tf.subtract(tf.ones_like(edges_mask), edges_mask)) # Define edges loss GNN['loss_edges'] = tf.losses.sigmoid_cross_entropy( multi_class_labels = edges_mask, logits = E_vote, weights = tf.add( tf.scalar_mul( tf.divide(tf.add(pos_edges_n,neg_edges_n),pos_edges_n), edges_mask), tf.scalar_mul( tf.divide(tf.add(pos_edges_n,neg_edges_n),neg_edges_n), tf.subtract(tf.ones_like(edges_mask), edges_mask) ) ) ) # Define decision loss GNN['loss_decision'] = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=route_exists, logits=pred_logits)) # Compute true positives, false positives, true negatives, false negatives GNN['true_pos'] = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32))) GNN['false_pos'] = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32))) GNN['true_neg'] = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32))) GNN['false_neg'] = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32))) # Define edges accuracy GNN['acc_edges'] = tf.reduce_mean(tf.cast(tf.equal(edges_mask, tf.round(E_prob)), tf.float32)) # Define decision accuracy GNN['acc_decision'] = tf.reduce_mean(tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)) # Define optimizer optimizer = tf.train.AdamOptimizer(name='Adam', learning_rate=learning_rate) # Compute cost relative to L2 normalization vars_cost = tf.add_n([ tf.nn.l2_loss(var) for var in tf.trainable_variables() ]) # Define gradients and train step for loss_type in ['edges','decision']: grads, _ = tf.clip_by_global_norm(tf.gradients(GNN['loss_' + loss_type] + tf.multiply(vars_cost, l2norm_scaling),tf.trainable_variables()),global_norm_gradient_clipping_ratio) GNN['train_step_' + loss_type] = optimizer.apply_gradients(zip(grads, tf.trainable_variables())) #end # Return GNN dictionary return GNN
def _static_subsample(self, indicator, batch_size, labels): """Returns subsampled minibatch. Args: indicator: boolean tensor of shape [N] whose True entries can be sampled. N should be a complie time constant. batch_size: desired batch size. This scalar cannot be None. labels: boolean tensor of shape [N] denoting positive(=True) and negative (=False) examples. N should be a complie time constant. Returns: sampled_idx_indicator: boolean tensor of shape [N], True for entries which are sampled. It ensures the length of output of the subsample is always batch_size, even when number of examples set to True in indicator is less than batch_size. Raises: ValueError: if labels and indicator are not 1D boolean tensors. """ # Check if indicator and labels have a static size. if not indicator.shape.is_fully_defined(): raise ValueError( 'indicator must be static in shape when is_static is' 'True') if not labels.shape.is_fully_defined(): raise ValueError('labels must be static in shape when is_static is' 'True') if not isinstance(batch_size, int): raise ValueError( 'batch_size has to be an integer when is_static is' 'True.') input_length = tf.shape(indicator)[0] # Set the number of examples set True in indicator to be at least # batch_size. num_true_sampled = tf.reduce_sum(tf.cast(indicator, tf.float32)) additional_false_sample = tf.less_equal( tf.cumsum(tf.cast(tf.logical_not(indicator), tf.float32)), batch_size - num_true_sampled) indicator = tf.logical_or(indicator, additional_false_sample) # Shuffle indicator and label. Need to store the permutation to restore the # order post sampling. permutation = tf.random_shuffle(tf.range(input_length)) indicator = ops.matmul_gather_on_zeroth_axis( tf.cast(indicator, tf.float32), permutation) labels = ops.matmul_gather_on_zeroth_axis(tf.cast(labels, tf.float32), permutation) # index (starting from 1) when indicator is True, 0 when False indicator_idx = tf.where(tf.cast(indicator, tf.bool), tf.range(1, input_length + 1), tf.zeros(input_length, tf.int32)) # Replace -1 for negative, +1 for positive labels signed_label = tf.where( tf.cast(labels, tf.bool), tf.ones(input_length, tf.int32), tf.scalar_mul(-1, tf.ones(input_length, tf.int32))) # negative of index for negative label, positive index for positive label, # 0 when indicator is False. signed_indicator_idx = tf.multiply(indicator_idx, signed_label) sorted_signed_indicator_idx = tf.nn.top_k(signed_indicator_idx, input_length, sorted=True).values [num_positive_samples, num_negative_samples ] = self._get_num_pos_neg_samples(sorted_signed_indicator_idx, batch_size) sampled_idx = self._get_values_from_start_and_end( sorted_signed_indicator_idx, num_positive_samples, num_negative_samples, batch_size) # Shift the indices to start from 0 and remove any samples that are set as # False. sampled_idx = tf.abs(sampled_idx) - tf.ones(batch_size, tf.int32) sampled_idx = tf.multiply( tf.cast(tf.greater_equal(sampled_idx, tf.constant(0)), tf.int32), sampled_idx) sampled_idx_indicator = tf.cast( tf.reduce_sum(tf.one_hot(sampled_idx, depth=input_length), axis=0), tf.bool) # project back the order based on stored permutations reprojections = tf.one_hot(permutation, depth=input_length, dtype=tf.float32) return tf.cast( tf.tensordot(tf.cast(sampled_idx_indicator, tf.float32), reprojections, axes=[0, 0]), tf.bool)
def add_prediction_op(self): """ Adds the unrolled RNN: h_0 = 0 for t in 1 to T: o_t, h_t = cell(x_t, h_{t-1}) o_drop_t = Dropout(o_t, dropout_rate) y_t = o_drop_t U + b_2 Returns: pred: tf.Tensor of shape (batch_size, max_length, non_terminal_vocab) """ x = self.add_embedding() dropout_rate = self.dropout_placeholder preds = [] # Predicted output at each timestep should go here! hidden = [] cell = LSTMCell(Config.n_token_features * Config.embed_size, Config.hidden_size) # Define U and b2 as variables. # Initialize state as vector of zeros. xinit = tf.contrib.layers.xavier_initializer(dtype=tf.float64) if not self.config.terminal_pred: output_size = self.config.non_terminal_vocab else: output_size = self.config.terminal_vocab U = tf.get_variable('U', shape=[self.config.hidden_size, output_size], initializer=xinit, dtype=tf.float64) b2 = tf.get_variable('b2', shape=[output_size], initializer=tf.constant_initializer(0.0), dtype=tf.float64) c_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size], dtype=tf.float64) h_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size], dtype=tf.float64) state_tuple = (c_t, h_t) scope = "LSTM_terminal" if self.config.terminal_pred else "LSTM_non_terminal" with tf.variable_scope(scope): for time_step in range(self.max_length): if time_step > 0: tf.get_variable_scope().reuse_variables() o_t, h_t = cell(x[:, time_step, :], state_tuple) o_drop_t = tf.nn.dropout(o_t, dropout_rate) preds.append(tf.matmul(o_drop_t, U) + b2) hidden.append(h_t[1]) if not (self.config.cell == "lstmAend") and not (self.config.cell == "lstm"): W_a = tf.get_variable('W_a', shape=[ self.config.hidden_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) W_o = tf.get_variable( 'W_o', shape=[2 * self.config.hidden_size, output_size], dtype=tf.float64, initializer=xinit) W_s = tf.get_variable('W_s', shape=[output_size, output_size], dtype=tf.float64, initializer=xinit) b_o = tf.get_variable( 'b_o', shape=[output_size], dtype=tf.float64, initializer=tf.constant_initializer(0.0)) b_s = tf.get_variable( 'b_s', shape=[output_size], dtype=tf.float64, initializer=tf.constant_initializer(0.0)) hidden_stack = tf.stack(hidden, 1) ht = tf.reshape( tf.matmul(h_t[1], W_a), (tf.shape(x)[0], -1, self.config.hidden_size)) weights = tf.reduce_sum( ht * hidden_stack, axis=2) * tf.slice( self.attn_mask_placeholder, [0, 0], [-1, time_step + 1]) weights = tf.nn.softmax(weights) context = tf.reduce_sum(tf.reshape( weights, (tf.shape(weights)[0], tf.shape(weights)[1], -1)) * hidden_stack, axis=1) context_hidden_sum = tf.add(context, h_t[1]) if (self.config.cell == "lstmAcont"): hidden = hidden[:-1] + [context] if (self.config.cell == "lstmAsum_fn"): W_alpha = tf.get_variable('W_alpha', shape=[ self.config.hidden_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) W_beta = tf.get_variable('W_beta', shape=[ self.config.hidden_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) U_alpha = tf.get_variable('U_alpha', shape=[ self.config.embed_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) U_beta = tf.get_variable('U_beta', shape=[ self.config.embed_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) alpha = tf.sigmoid( tf.matmul(context, W_alpha) + tf.matmul(x[:, time_step, :], U_alpha)) beta = tf.sigmoid( tf.matmul(h_t[1], W_beta) + tf.matmul(x[:, time_step, :], U_beta)) straightSum = alpha * context + beta * h_t[1] hidden = hidden[:-1] + [straightSum] if (self.config.cell == "lstmAsum"): alpha = tf.get_variable( 'alpha', shape=(), dtype=tf.float64, initializer=tf.random_uniform_initializer( -1.0, 2.0)) beta = tf.get_variable( 'beta', shape=(), dtype=tf.float64, initializer=tf.random_uniform_initializer( -1.0, 2.0)) straightSum = tf.add(tf.scalar_mul(alpha, context), tf.scalar_mul(beta, h_t[1])) hidden = hidden[:-1] + [straightSum] if (self.config.cell == "lstmAwsum_fn"): W_ph = tf.get_variable('W_ph', shape=[ self.config.hidden_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) W_pc = tf.get_variable('W_pc', shape=[ self.config.hidden_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) W_px = tf.get_variable('W_px', shape=[ self.config.embed_size, self.config.hidden_size ], dtype=tf.float64, initializer=xinit) hTerm = tf.matmul(h_t[1], W_ph) cTerm = tf.matmul(context, W_pc) xTerm = tf.matmul(x[:, time_step, :], W_px) p_arr = tf.sigmoid(hTerm + cTerm + xTerm) weightedSum = (p_arr * context) + ( (1 - p_arr) * h_t[1]) hidden = hidden[:-1] + [weightedSum] preds = tf.stack(preds, 1) hidden = tf.stack(hidden, 1) final_preds = tf.boolean_mask(preds, self.mask_placeholder) final_hidden = tf.boolean_mask(hidden, self.mask_placeholder) if not (self.config.cell == "lstm"): W_a = tf.get_variable( 'W_a', shape=[self.config.hidden_size, self.config.hidden_size], dtype=tf.float64, initializer=xinit) W_o = tf.get_variable( 'W_o', shape=[2 * self.config.hidden_size, output_size], dtype=tf.float64, initializer=xinit) W_s = tf.get_variable('W_s', shape=[output_size, output_size], dtype=tf.float64, initializer=xinit) b_o = tf.get_variable('b_o', shape=[output_size], dtype=tf.float64, initializer=tf.constant_initializer(0.0)) b_s = tf.get_variable('b_s', shape=[output_size], dtype=tf.float64, initializer=tf.constant_initializer(0.0)) ht = tf.reshape(tf.matmul(final_hidden, W_a), (tf.shape(x)[0], -1, self.config.hidden_size)) weights = tf.reduce_sum(ht * hidden, axis=2) * self.attn_mask_placeholder weights = tf.nn.softmax(weights) context = tf.reduce_sum( tf.reshape(weights, (tf.shape(weights)[0], tf.shape(weights)[1], -1)) * hidden, axis=1) final_preds = tf.tanh( tf.matmul(tf.concat(1, [context, final_hidden]), W_o) + b_o) final_preds = tf.matmul(final_preds, W_s) + b_s if self.config.terminal_pred: nt = tf.nn.embedding_lookup( self.embeddings, self.next_non_terminal_input_placeholder) nt = tf.reshape( nt, [-1, self.config.n_token_features * self.config.embed_size]) U_nt = tf.get_variable( 'U_nt', shape=[self.config.hidden_size, output_size], initializer=xinit, dtype=tf.float64) b_t = tf.get_variable('b_t', shape=[output_size], initializer=tf.constant_initializer(0.0), dtype=tf.float64) final_preds = final_preds + tf.matmul(nt, U_nt) + b_t return final_preds
def fit(self, mu, x, y_desired): y_pred = self(x) e = y_desired - y_pred x = tf.constant(np.array([1.0] + x, dtype=np.float32)) self.w.assign_add(tf.scalar_mul(2 * mu * e, x))
def main(): """Create the model and start the training.""" args = get_arguments() """ Get configurations here. We pass some arguments from command line to init configurations, for training hyperparameters, you can set them in TrainConfig Class. Note: we set filter scale to 1 for pruned model, 2 for non-pruned model. The filters numbers of non-pruned model is two times larger than prunde model, e.g., [h, w, 64] <-> [h, w, 32]. """ cfg = TrainConfig(dataset=args.dataset, is_training=True, random_scale=args.random_scale, random_mirror=args.random_mirror, filter_scale=args.filter_scale) if args.num_classes is not None: cfg.param["num_classes"] = args.num_classes if args.data_dir is not None: cfg.param["data_dir"] = args.data_dir if args.val_list is not None: cfg.param["eval_list"] = args.val_list if args.train_list is not None: cfg.param["train_list"] = args.train_list if args.ignore_label is not None: cfg.param["ignore_label"] = args.ignore_label if args.eval_size is not None: cfg.param["eval_size"] = [ int(x.strip()) for x in args.eval_size.split("x")[::-1] ] if args.training_size is not None: cfg.TRAINING_SIZE = [ int(x.strip()) for x in args.training_size.split("x")[::-1] ] if args.batch_size is not None: cfg.BATCH_SIZE = args.batch_size if args.learning_rate is not None: cfg.LEARNING_RATE = args.learning_rate if args.restore_from is not None: cfg.model_weight = args.restore_from if args.snapshot_dir is not None: cfg.SNAPSHOT_DIR = args.snapshot_dir if args.restore_from == "scratch": from tqdm import tqdm import cv2 import joblib as joblib if not args.img_mean: print( "Calculating img mean for custom dataset. To prevent this, specify it with --img-mean next time" ) image_files, annotation_files = read_labeled_image_list( cfg.param["data_dir"], cfg.param["train_list"]) means = joblib.Parallel(n_jobs=6)( joblib.delayed(calc_mean)(image_file, cv2) for image_file in tqdm(image_files, desc="calc img mean")) cfg.IMG_MEAN = np.mean(means, axis=0).tolist() else: cfg.IMG_MEAN = [float(x.strip()) for x in args.img_mean.split(",")] cfg.display() # Setup training network and training samples train_reader = ImageReader(cfg=cfg, mode='train') train_net = ICNet_BN(image_reader=train_reader, cfg=cfg, mode='train') loss_sub4, loss_sub24, loss_sub124, reduced_loss = create_losses( train_net, train_net.labels, cfg) # Setup validation network and validation samples with tf.variable_scope('', reuse=True): val_reader = ImageReader(cfg, mode='eval') val_net = ICNet_BN(image_reader=val_reader, cfg=cfg, mode='train') val_loss_sub4, val_loss_sub24, val_loss_sub124, val_reduced_loss = create_losses( val_net, val_net.labels, cfg) # Using Poly learning rate policy base_lr = tf.constant(cfg.LEARNING_RATE) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - step_ph / cfg.TRAINING_STEPS), cfg.POWER)) # Set restore variable restore_var = tf.global_variables() all_trainable = [ v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma ] # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, cfg.MOMENTUM) grads = tf.gradients(reduced_loss, all_trainable) train_op = opt_conv.apply_gradients(zip(grads, all_trainable)) # Create session & restore weights (Here we only need to use train_net to create session since we reuse it) train_net.create_session() if args.initializer: train_net.set_initializer(initializer_algorithm=args.initializer) train_net.initialize_variables() if not args.restore_from or args.restore_from != "scratch": train_net.restore(cfg.model_weight, restore_var) saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=20) total_parameters = 0 for variable in tf.trainable_variables(): # shape is an array of tf.Dimension shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters print("Total trainable parameters: " + str(total_parameters)) # Iterate over training steps. val_loss_value = 10.0 min_val_loss = float("inf") stagnation = 0 max_non_decreasing_val_loss = int( np.ceil(args.early_stopping_patience * len(train_reader.image_list) / (cfg.BATCH_SIZE * cfg.EVAL_EVERY))) print( "Maximum times that val loss can stagnate before early stopping is applied: " + str(max_non_decreasing_val_loss)) for step in range(cfg.TRAINING_STEPS): start_time = time.time() feed_dict = {step_ph: step} if step % cfg.EVAL_EVERY == 0: loss_value, loss1, loss2, loss3, val_loss_value, _ = train_net.sess.run( [ reduced_loss, loss_sub4, loss_sub24, loss_sub124, val_reduced_loss, train_op ], feed_dict=feed_dict) if val_loss_value < min_val_loss: print("New best val loss {:.3f}. Saving weights...".format( val_loss_value)) train_net.save( saver, cfg.SNAPSHOT_DIR, step, model_name="val{:.3f}model.ckpt".format(val_loss_value)) min_val_loss = val_loss_value stagnation = 0 else: stagnation += 1 else: loss_value, loss1, loss2, loss3, _ = train_net.sess.run( [reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f}, val_loss: {:.3f} ({:.3f} sec/step)'.\ format(step, loss_value, loss1, loss2, loss3, val_loss_value, duration)) if stagnation > max_non_decreasing_val_loss: print("Early stopping") break
def floatToFixPoint(tensor_): tensor_ = tf.clip_by_value(tensor_,-32.0,32.0) tensor_ = tf.scalar_mul(67108864.0,tensor_) tensor_ = tf.round(tensor_) tensor_ = tf.scalar_mul(1/67108864.0,tensor_) return (tensor_)
def mnist_fid_score(self, epoch, fid_train_images_names): def data_reader_faces(filename): with tf.device('/CPU'): print(tf.cast(filename[0], dtype=tf.string)) image_string = tf.io.read_file( tf.cast(filename[0], dtype=tf.string)) # Don't use tf.image.decode_image, or the output shape will be undefined image = tf.image.decode_jpeg(image_string, channels=3) image.set_shape([218, 178, 3]) image = tf.image.crop_to_bounding_box(image, 38, 18, 140, 140) image = tf.image.resize(image, [80, 80]) # This will convert to float values in [0, 1] image = tf.subtract(image, 127.5) image = tf.divide(image, 127.5) return image self.load_fid_model() fid_num_samples = 1000 fid_batch_size = tf.constant(100, dtype='int64') num_parallel_calls = 4 random_points = tf.keras.backend.random_uniform( [min(fid_train_images_names.shape[0], fid_num_samples)], minval=0, maxval=int(fid_train_images_names.shape[0]), dtype='int32', seed=None) fid_train_images_names = fid_train_images_names[random_points] fid_image_dataset = tf.data.Dataset.from_tensor_slices( fid_train_images_names) fid_image_dataset = fid_image_dataset.map( data_reader_faces, num_parallel_calls=int(num_parallel_calls)) fid_image_dataset = fid_image_dataset.batch(fid_batch_size) with tf.device(config.DEVICE): count = 0 fid_sum = 0 for image_batch in fid_image_dataset: noise = tf.random.normal([fid_batch_size, self.noise_dim], self.noise_mean, self.noise_stddev) preds = self.generator(noise, training=False) preds = tf.image.resize(preds, [80, 80]) preds = tf.scalar_mul(2., preds) preds = tf.subtract(preds, 1.0) preds = preds.numpy() act1 = self.fid_model.predict(image_batch) act2 = self.fid_model.predict(preds) try: act1 = np.concatenate((act1, act1), axis=0) act2 = np.concatenate((act2, act2), axis=0) fid_score = self.calculate_fid(act1, act2) fid_sum += fid_score count += 1 except: act1 = act1 act2 = act2 avg_fid_score = fid_sum / count / fid_batch_size self.fid_scores.append( str(epoch) + ',' + str(np.array(avg_fid_score))) print("epoch: %d fid: %f" % (epoch, avg_fid_score))
def train_setup(self): tf.set_random_seed(self.conf.random_seed) # Create queue coordinator. self.coord = tf.train.Coordinator() # Input size h, w = (self.conf.input_height, self.conf.input_width) input_size = (h, w) # Devices gpu_list = get_available_gpus() zip_encoder, zip_decoder_b, zip_decoder_w, zip_crf = [], [], [], [] previous_crf_names = [] restore_vars = [] self.loaders = [] self.im_list = [] for i in range(len(gpu_list)): with tf.device(gpu_list[i]): # Load reader with tf.name_scope("create_inputs"): reader = ImageReader(self.conf.data_dir, self.conf.data_list, input_size, self.conf.random_scale, self.conf.random_mirror, self.conf.ignore_label, IMG_MEAN, self.coord) self.image_batch, self.label_batch = reader.dequeue( self.conf.batch_size) self.im_list.append(self.image_batch) image_batch_075 = tf.image.resize_images( self.image_batch, [int(h * 0.75), int(w * 0.75)]) image_batch_05 = tf.image.resize_images( self.image_batch, [int(h * 0.5), int(w * 0.5)]) # Create network with tf.variable_scope('', reuse=False): net = Deeplab_v2(self.image_batch, self.conf.num_classes, True, rescale075=False, rescale05=False, crf_type=self.conf.crf_type) with tf.variable_scope('', reuse=True): net075 = Deeplab_v2(image_batch_075, self.conf.num_classes, True, rescale075=True, rescale05=False, crf_type=self.conf.crf_type) with tf.variable_scope('', reuse=True): net05 = Deeplab_v2(image_batch_05, self.conf.num_classes, True, rescale075=False, rescale05=True, crf_type=self.conf.crf_type) # Variables that load from pre-trained model. restore_var = [ v for v in tf.global_variables() if ('fc' not in v.name and 'crfrnn' not in v.name) ] restore_vars.append(restore_var) # Trainable Variables all_trainable = tf.trainable_variables() # Fine-tune part for name in previous_crf_names: for v in all_trainable: if v.name == name: all_trainable.remove(v) crf_trainable = [ v for v in all_trainable if ('crfrnn' in v.name and v.name not in previous_crf_names ) ] previous_crf_names.extend(v.name for v in crf_trainable) encoder_trainable = [ v for v in all_trainable if 'fc' not in v.name and 'crfrnn' not in v.name ] # lr * 1.0 # Remove encoder_trainable from all_trainable #all_trainable = [v for v in all_trainable if v not in encoder_trainable] # Decoder part decoder_trainable = [ v for v in all_trainable if 'fc' in v.name and 'crfrnn' not in v.name ] decoder_w_trainable = [ v for v in decoder_trainable if ('weights' in v.name or 'gamma' in v.name) and 'crfrnn' not in v.name ] # lr * 10.0 decoder_b_trainable = [ v for v in decoder_trainable if ('biases' in v.name or 'beta' in v.name) and 'crfrnn' not in v.name ] # lr * 20.0 # Check assert (len(all_trainable) == len(decoder_trainable) + len(crf_trainable)) + len(encoder_trainable) assert (len(decoder_trainable) == len(decoder_w_trainable) + len(decoder_b_trainable)) # Network raw output raw_output100 = net.outputs raw_output075 = net075.outputs raw_output05 = net05.outputs raw_output = tf.reduce_max(tf.stack([ raw_output100, tf.image.resize_images(raw_output075, tf.shape(raw_output100)[1:3, ]), tf.image.resize_images(raw_output05, tf.shape(raw_output100)[1:3, ]) ]), axis=0) # Ground Truth: ignoring all labels greater or equal than n_classes label_proc = prepare_label(self.label_batch, tf.stack( raw_output.get_shape()[1:3]), num_classes=self.conf.num_classes, one_hot=True) # [batch_size, h, w] label_proc075 = prepare_label( self.label_batch, tf.stack(raw_output075.get_shape()[1:3]), num_classes=self.conf.num_classes, one_hot=True) label_proc05 = prepare_label( self.label_batch, tf.stack(raw_output05.get_shape()[1:3]), num_classes=self.conf.num_classes, one_hot=True) raw_gt = tf.reshape(label_proc, [ -1, ]) raw_gt075 = tf.reshape(label_proc075, [ -1, ]) raw_gt05 = tf.reshape(label_proc05, [ -1, ]) indices = tf.squeeze( tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)), 1) indices075 = tf.squeeze( tf.where( tf.less_equal(raw_gt075, self.conf.num_classes - 1)), 1) indices05 = tf.squeeze( tf.where(tf.less_equal(raw_gt05, self.conf.num_classes - 1)), 1) gt = tf.cast(tf.gather(raw_gt, indices), tf.int32) gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32) gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32) raw_prediction = tf.reshape(raw_output, [-1, self.conf.num_classes]) raw_prediction100 = tf.reshape(raw_output100, [-1, self.conf.num_classes]) raw_prediction075 = tf.reshape(raw_output075, [-1, self.conf.num_classes]) raw_prediction05 = tf.reshape(raw_output05, [-1, self.conf.num_classes]) prediction = tf.gather(raw_prediction, indices) prediction100 = tf.gather(raw_prediction100, indices) prediction075 = tf.gather(raw_prediction075, indices075) prediction05 = tf.gather(raw_prediction05, indices05) # Pixel-wise softmax_cross_entropy loss #loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt) loss = tf.nn.softmax_cross_entropy_with_logits_v2( logits=raw_prediction, labels=tf.reshape(label_proc[0], (h * w, self.conf.num_classes))) ''' coefficients = [0.01460247, 1.25147725, 2.88479363, 1.20348121, 1.65261654, 1.67514772, 0.62338799, 0.7729363, 0.42038501, 0.98557268, 1.31867536, 0.85313332, 0.67227604, 1.21317965, 1. , 0.24263748, 1.80877607, 1.3082213, 0.79664027, 0.72543945, 1.27823374] ''' #loss = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc[0], (h*w, self.conf.num_classes)), logits=raw_prediction) #loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction100, labels=gt) loss100 = tf.nn.softmax_cross_entropy_with_logits_v2( logits=raw_prediction100, labels=tf.reshape(label_proc[0], (h * w, self.conf.num_classes))) #loss100 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc[0], (h*w, self.conf.num_classes)), logits=raw_prediction100) #loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction075, labels=gt075) loss075 = tf.nn.softmax_cross_entropy_with_logits_v2( logits=raw_prediction075, labels=tf.reshape(label_proc075[0], (int(h * 0.75) * int(w * 0.75), self.conf.num_classes))) #loss075 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc075[0], (int(h * 0.75) * int(w * 0.75), self.conf.num_classes)), logits=raw_prediction075) #loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction05, labels=gt05) loss05 = tf.nn.softmax_cross_entropy_with_logits_v2( logits=raw_prediction05, labels=tf.reshape( label_proc05[0], (int(h * 0.5) * int(w * 0.5), self.conf.num_classes))) #loss05 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc05[0], (int(h * 0.5) * int(w * 0.5), self.conf.num_classes)), logits=raw_prediction05) # L2 regularization l2_losses = [ self.conf.weight_decay * tf.nn.l2_loss(v) for v in all_trainable if 'weights' in v.name ] # Loss function self.reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean( loss100) + tf.reduce_mean(loss075) + tf.reduce_mean( loss05) + tf.add_n(l2_losses) # Define optimizers # 'poly' learning rate base_lr = tf.constant(self.conf.learning_rate) self.curr_step = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul( base_lr, tf.pow((1 - self.curr_step / self.conf.num_steps), self.conf.power)) # We have several optimizers here in order to handle the different lr_mult # which is a kind of parameters in Caffe. This controls the actual lr for each # layer. opt_encoder = tf.train.MomentumOptimizer( learning_rate, self.conf.momentum) opt_decoder_w = tf.train.MomentumOptimizer( learning_rate * 10.0, self.conf.momentum) opt_decoder_b = tf.train.MomentumOptimizer( learning_rate * 20.0, self.conf.momentum) opt_crf = tf.train.MomentumOptimizer(learning_rate, self.conf.momentum) # Gradient accumulation # Define a variable to accumulate gradients. accum_grads = [ tf.Variable(tf.zeros_like(v.initialized_value()), trainable=False) for v in encoder_trainable + decoder_w_trainable + decoder_b_trainable + crf_trainable ] # Define an operation to clear the accumulated gradients for next batch. self.zero_op = [ v.assign(tf.zeros_like(v)) for v in accum_grads ] # To make sure each layer gets updated by different lr's, we do not use 'minimize' here. # Instead, we separate the steps compute_grads+update_params. # Compute grads grads = tf.gradients( self.reduced_loss, encoder_trainable + decoder_w_trainable + decoder_b_trainable + crf_trainable) # Accumulate and normalise the gradients. self.accum_grads_op = [ accum_grads[i].assign_add(grad / self.conf.grad_update_every) for i, grad in enumerate(grads) ] grads_encoder = accum_grads[:len(encoder_trainable)] grads_decoder_w = accum_grads[len(encoder_trainable ):len(encoder_trainable) + len(decoder_w_trainable)] grads_decoder_b = accum_grads[( len(encoder_trainable) + len(decoder_w_trainable)):(len(encoder_trainable) + len(decoder_w_trainable) + len(decoder_b_trainable))] grads_crf = accum_grads[ len(encoder_trainable) + len(decoder_w_trainable) + len(decoder_b_trainable ):] # assuming crf gradients are appended to the end zip_encoder.append(list(zip(grads_encoder, encoder_trainable))) zip_decoder_b.append( list(zip(grads_decoder_b, decoder_b_trainable))) zip_decoder_w.append( list(zip(grads_decoder_w, decoder_w_trainable))) zip_crf.append(list(zip(grads_crf, crf_trainable))) avg_grads_encoder = average_gradients(zip_encoder) avg_grads_decoder_w = average_gradients(zip_decoder_w) avg_grads_decoder_b = average_gradients(zip_decoder_b) avg_grads_crf = average_gradients(zip_crf) for i in range(len(gpu_list)): with tf.device(gpu_list[i]): # Update params train_op_conv = opt_encoder.apply_gradients(avg_grads_encoder) train_op_fc_w = opt_decoder_w.apply_gradients( avg_grads_decoder_w) train_op_fc_b = opt_decoder_b.apply_gradients( avg_grads_decoder_b) train_op_crf = opt_crf.apply_gradients(avg_grads_crf) # Finally, get the train_op! update_ops = tf.get_collection( tf.GraphKeys.UPDATE_OPS ) # for collecting moving_mean and moving_variance with tf.control_dependencies(update_ops): self.train_op = tf.group(train_op_fc_w, train_op_fc_b, train_op_crf) # train_op_conv # Saver for storing checkpoints of the model self.saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=0) # Loader for loading the pre-trained model for i in range(len(gpu_list)): with tf.device(gpu_list[i]): self.loaders.append(tf.train.Saver(var_list=restore_vars[i])) #self.loaders.append(tf.train.Saver(var_list=tf.global_variables())) # Training summary # Processed predictions: for visualisation. raw_output_up = tf.image.resize_bilinear(raw_output, input_size) raw_output_up = tf.argmax(raw_output_up, axis=3) self.pred = tf.expand_dims(raw_output_up, axis=3) # Image summary. images_summary = tf.py_func(inv_preprocess, [self.image_batch, 1, IMG_MEAN], tf.uint8) labels_summary = tf.py_func( decode_labels, [self.label_batch, 1, self.conf.num_classes], tf.uint8) preds_summary = tf.py_func(decode_labels, [self.pred, 1, self.conf.num_classes], tf.uint8) self.total_summary = tf.summary.image( 'images', tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]), max_outputs=1) # Concatenate row-wise. if not os.path.exists(self.conf.logdir): os.makedirs(self.conf.logdir) self.summary_writer = tf.summary.FileWriter( self.conf.logdir, graph=tf.get_default_graph())
def preprocess_for_train(image, classes, boxes, resolution, speed_mode=False): if speed_mode: pass else: image_ori = image classes_ori = classes boxes_ori = boxes # randomly sample patches if RANDOM_SUB_SAMPLE: image_shape = tf.shape(image) x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1) x1 = tf.expand_dims(x1, -1) x2 = tf.expand_dims(x2, -1) y1 = tf.expand_dims(y1, -1) y2 = tf.expand_dims(y2, -1) boxes = tf.concat([y1, x1, y2, x2], axis=1) bbox_begin, bbox_size, distort_bbox = tf.image.sample_distorted_bounding_box( image_shape, bounding_boxes=tf.expand_dims(boxes, 0), min_object_covered=BBOX_CROP_OVERLAP, aspect_ratio_range=(0.5, 2.0), area_range=(0.3, 1.0), max_attempts=200, use_image_if_no_bounding_boxes=True) distort_bbox = distort_bbox[0, 0] image = tf.slice(image, bbox_begin, bbox_size) boxes = bboxes_resize(distort_bbox, boxes) classes, boxes = bboxes_filter_overlap(classes, boxes, threshold=BBOX_CROP_OVERLAP, assign_negative=False) y1, x1, y2, x2 = tf.unstack(boxes, 4, axis=1) image_shape = tf.shape(image) x1 = tf.clip_by_value(x1, 0.0, 1.0) x2 = tf.clip_by_value(x2, 0.0, 1.0) y1 = tf.clip_by_value(y1, 0.0, 1.0) y2 = tf.clip_by_value(y2, 0.0, 1.0) x1 = tf.expand_dims(x1, -1) x2 = tf.expand_dims(x2, -1) y1 = tf.expand_dims(y1, -1) y2 = tf.expand_dims(y2, -1) boxes = tf.concat([x1, y1, x2, y2], axis=1) if RANDOM_ZOOM_OUT: image, boxes = random_zoom_out(image, boxes) if RANDOM_HFLIP: # Randomly flip the image horizontally. image, boxes = random_flip_left_right(image, boxes) if RANDOM_COLOR: # Color distortion image = tf.div(image, 255.0) image = apply_with_random_selector( image, lambda x, ordering: distort_color(x, ordering, fast_mode=False ), num_cases=4) image = tf.scalar_mul(255.0, image) # Rollback to the full image if there is no boxes no_box_cond = tf.equal(tf.size(boxes), 0) image = tf.cond(no_box_cond, lambda: image_ori, lambda: image) boxes = tf.cond(no_box_cond, lambda: boxes_ori, lambda: boxes) classes = tf.cond(no_box_cond, lambda: classes_ori, lambda: classes) # Scaling to canonical size without perspective preserved image, scale, translation = aspect_preserving_resize( image, resolution, depth=3, resize_mode="bilinear") new_image = tf.image.resize_image_with_crop_or_pad( image, resolution, resolution) scale_x = tf.to_float(tf.shape(image)[1]) / tf.to_float(resolution) scale_y = tf.to_float(tf.shape(image)[0]) / tf.to_float(resolution) shift_x = tf.math.maximum( 0.0, tf.to_float(resolution - tf.shape(image)[1]) / float(resolution * 2)) shift_y = tf.math.maximum( 0.0, tf.to_float(resolution - tf.shape(image)[0]) / float(resolution * 2)) x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1) x1 = tf.scalar_mul(scale_x, x1) y1 = tf.scalar_mul(scale_y, y1) x2 = tf.scalar_mul(scale_x, x2) y2 = tf.scalar_mul(scale_y, y2) boxes = tf.concat([ tf.expand_dims(x1, -1), tf.expand_dims(y1, -1), tf.expand_dims(x2, -1), tf.expand_dims(y2, -1) ], axis=1) boxes = boxes + [shift_x, shift_y, shift_x, shift_y] image = new_image # Scaling to canonical size without preserving perspective # image, scale, translation = bilinear_resize(image, resolution, depth=3, resize_mode="bilinear") # mean subtraction means = [_R_MEAN, _G_MEAN, _B_MEAN] channels = tf.split(axis=2, num_or_size_splits=3, value=image) for i in range(3): channels[i] -= means[i] # image = tf.concat(axis=2, values=channels) # caffe swaps color channels image = tf.concat(axis=2, values=[channels[2], channels[1], channels[0]]) return image, classes, boxes, scale, translation
def main(): """Create the model and start the training.""" args = get_arguments() h, w = map(int, args.input_size.split(',')) input_size = (h, w) coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = ImageReader( DATA_DIR, DATA_LIST_PATH, input_size, args.random_scale, args.random_mirror, args.ignore_label, IMG_MEAN, coord) image_batch, label_batch = reader.dequeue(args.batch_size) net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale) sub4_out = net.layers['sub4_out'] sub24_out = net.layers['sub24_out'] sub124_out = net.layers['conv6_cls'] restore_var = tf.global_variables() # restore_var = [v for v in tf.global_variables() if 'conv6_cls' not in v.name] all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma] loss_sub4 = create_loss(sub4_out, label_batch, args.num_classes, args.ignore_label) loss_sub24 = create_loss(sub24_out, label_batch, args.num_classes, args.ignore_label) loss_sub124 = create_loss(sub124_out, label_batch, args.num_classes, args.ignore_label) l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name] reduced_loss = LAMBDA1 * loss_sub4 + LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses) # Using Poly learning rate policy base_lr = tf.constant(args.learning_rate) step_ph = tf.placeholder(dtype=tf.float32, shape=()) learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS if args.update_mean_var == False: update_ops = None else: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum) grads = tf.gradients(reduced_loss, all_trainable) train_op = opt_conv.apply_gradients(zip(grads, all_trainable)) # Set up tf session and initialize variables. # 先初始化所有变量 config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init = tf.global_variables_initializer() sess.run(init) # Saver for storing checkpoints of the model. saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=5) ckpt = tf.train.get_checkpoint_state(args.snapshot_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('Restore from pre-trained model...') net.load(args.restore_from, sess) # args.restore_from:./model/icnet_cityscapes_trainval_90k_bnnomerge.npy # net_data = np.load(open(args.restore_from,"rb"),encoding="latin1").item() # loader = tf.train.Saver(var_list=exclude_restore_var) # loader.restore(sess, net_data) # Start queue threads. threads = tf.train.start_queue_runners(coord=coord, sess=sess) # Iterate over training steps. for step in range(args.num_steps): start_time = time.time() feed_dict = {step_ph: step} if step % args.save_pred_every == 0: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) save(saver, sess, args.snapshot_dir, step) else: loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict) duration = time.time() - start_time print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration)) coord.request_stop() coord.join(threads)
for i in range(args.train_batch): fake_labels[i, tmp_fake_labels[i]] = 1 loss = categorical_crossentropy(net.y_, net.y) top1 = categorical_accuracy(net.y_, net.y) top5 = top_k_categorical_accuracy(net.y_, net.y, 5) base_lr = 0.02 step = tf.Variable(0, trainable=False, name="Step") learning_rate = tf.train.exponential_decay(base_lr, step, 1, 0.999964) weight_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "W:0"] bias_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "b:0"] optimizer1 = tf.train.MomentumOptimizer(learning_rate, 0.9) optimizer2 = tf.train.MomentumOptimizer(tf.scalar_mul(2.0, learning_rate), 0.9) grads = optimizer1.compute_gradients(loss, var_list=weight_list+bias_list) w_grads = grads[:len(weight_list)] b_grads = grads[len(weight_list):] train1 = optimizer1.apply_gradients(w_grads, global_step=step) train2 = optimizer2.apply_gradients(b_grads, global_step=step) train_step = tf.group(train1, train2) init = tf.global_variables_initializer() sess = tf.Session() K.set_session(sess) sess.run(init) start = time.time()
dataset = tf.data.Dataset.from_tensor_slices((imNames, imLabels, imSplit)) dataset = dataset.map(parseFunction, num_parallel_calls=options.numParallelLoaders) dataset = dataset.shuffle(buffer_size=numItemsInDataset, seed=0) dataset = dataset.batch(options.batchSize) iterator = dataset.make_initializable_iterator() with tf.name_scope('Model'): # Data placeholders inputBatchImages, inputBatchImageNames, inputBatchImageLabels, inputBatchImageSplit = iterator.get_next( ) print("Data shape: %s" % str(inputBatchImages.get_shape())) if (options.modelName == "IncResV2") or (options.modelName == "NASNet"): scaledInputBatchImages = tf.scalar_mul((1.0 / 255), inputBatchImages) scaledInputBatchImages = tf.subtract(scaledInputBatchImages, 0.5) scaledInputBatchImages = tf.multiply(scaledInputBatchImages, 2.0) # Create model if options.modelName == "IncResV2": arg_scope = inception_resnet_v2.inception_resnet_v2_arg_scope() with slim.arg_scope(arg_scope): logits, aux_logits, endPoints = inception_resnet_v2.inception_resnet_v2( scaledInputBatchImages, is_training=False) # Get the lower layer and upper layer activations lowerLayerActivations = endPoints["?"] upperLayerActivations = endPoints["?"] # Create list of vars to restore before train op