def create_cost(c, w, b, nb, length, weight_spacing = 1.0, weight_bending = 1.0, gamma = 1.0, kappa = 2.0):
  #tangents
  t  = create_tangent(c);
  tn = create_normalize_tangent(t);
  nl = create_normal(tn);
  nr = tf.scalar_mul(-1.0, nl);
  
  l,r = create_left_right(c,w,nl);

  cost_left = create_cost_soft_min_aligned_distance(l, b, nl, nb, k = kappa, gamma = gamma);
  cost_right= create_cost_soft_min_aligned_distance(r, b, nr, nb, k = kappa, gamma = gamma);
  cost = tf.add(cost_left, cost_right);
  
  #spacing and bending
  if weight_spacing != 0:
    cost_spacing = tf.scalar_mul(weight_spacing, create_cost_spacing(t, length));
    cost = tf.add(cost, cost_spacing);
  else:
    cost_spacing = tf.constant(0);
  
  if weight_bending != 0:
    cost_bending = tf.scalar_mul(weight_bending, create_cost_bending(tn));
    cost = tf.add(cost, cost_bending);
  else:
    cost_bending = tf.constant(0);
  
  return (cost, cost_left, cost_right, cost_spacing, cost_bending, nl, l, r);
Exemplo n.º 2
0
def clip_norm(g, c, n):
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g
def predict_slim(sample_images, print_func=print):
    """
    Code modified from here: [https://github.com/tensorflow/models/issues/429]
    """
    # Setup preprocessing
    input_tensor = tf.placeholder(tf.float32, shape=(None, 299, 299, 3), name='input_image')
    scaled_input_tensor = tf.scalar_mul((1.0 / 255), input_tensor)
    scaled_input_tensor = tf.subtract(scaled_input_tensor, 0.5)
    scaled_input_tensor = tf.multiply(scaled_input_tensor, 2.0)

    # Setup session
    sess = tf.Session()
    arg_scope = slim_irv2.inception_resnet_v2_arg_scope()
    with slim.arg_scope(arg_scope):
        _, end_points = slim_irv2.inception_resnet_v2(scaled_input_tensor, is_training=False)

    # Load the model
    print_func("Loading TF-slim checkpoint...")
    saver = tf.train.Saver()
    saver.restore(sess, SLIM_CKPT)

    # Make prediction
    predict_values = []
    for image in sample_images:
        im = Image.open(image).resize((299, 299))
        arr = np.expand_dims(np.array(im), axis=0)
        y_pred = sess.run([end_points['Predictions']], feed_dict={input_tensor: arr})
        y_pred = y_pred[0].ravel()

        y_pred = y_pred[1:] / y_pred[1:].sum()  # remove background class and renormalize
        print_func("{} class={} prob={}".format(image, np.argmax(y_pred), np.max(y_pred)))
        predict_values.append(y_pred)

    return predict_values
Exemplo n.º 4
0
def thresholding(inputs):
    # find the mean for each example in the batch
    mean_output = tf.reduce_mean(inputs, axis=1)

    # scale each mean based on a factor
    threshold_scalar = tf.Variable(utils.threshold_scalar, tf.float32)
    scaled_mean = tf.scalar_mul(threshold_scalar, mean_output)
    scaled_mean = tf.reshape(scaled_mean, [utils.batch_size])

    # setup matrix for
    min_thresh_for_max = tf.fill([utils.batch_size], 0.05)
    max_thresh_for_min = tf.fill([utils.batch_size], 0.15)   #0.4
    thresholds = tf.maximum(min_thresh_for_max, scaled_mean)
    thresholds = tf.minimum(max_thresh_for_min, thresholds)

    # zero values under the thresholds using bitmask
    thresholds = tf.reshape(thresholds, [128, 1, 1])

    threshold_mask = tf.cast(tf.greater(inputs, thresholds), tf.float32)
    thresholded_input = tf.multiply(inputs, threshold_mask)

    # peak picking
    # select beats by x[i-1] < x[i] > x[i+1] (local maximum)
    x_minus_1 = tf.cast(tf.greater(thresholded_input, tf.manip.roll(thresholded_input, shift=-1, axis=1)), tf.float32)
    x_plus_1 = tf.cast(tf.greater(thresholded_input, tf.manip.roll(thresholded_input, shift=1, axis=1)), tf.float32)
    output = tf.multiply(x_minus_1, x_plus_1)


    return output
Exemplo n.º 5
0
  def focal_loss(onehot_labels, cls_preds,
                 alpha=0.25, gamma=2.0, name=None, scope=None):
    """Compute softmax focal loss between logits and onehot labels
    logits and onehot_labels must have same shape [batchsize, num_classes] and
    the same data type (float16, 32, 64)
    Args:
      onehot_labels: Each row labels[i] must be a valid probability distribution
      cls_preds: Unscaled log probabilities
      alpha: The hyperparameter for adjusting biased samples, default is 0.25
      gamma: The hyperparameter for penalizing the easy labeled samples
      name: A name for the operation (optional)
    Returns:
      A 1-D tensor of length batch_size of same type as logits with softmax focal loss
    """
    with tf.name_scope(scope, 'focal_loss', [cls_preds, onehot_labels]) as sc:
      logits = tf.convert_to_tensor(cls_preds)
      onehot_labels = tf.convert_to_tensor(onehot_labels)

      precise_logits = tf.cast(logits, tf.float32) if (
        logits.dtype == tf.float16) else logits
      onehot_labels = tf.cast(onehot_labels, precise_logits.dtype)
      predictions = tf.nn.sigmoid(logits)
      predictions_pt = tf.where(tf.equal(onehot_labels, 1), predictions, 1. - predictions)
      # add small value to avoid 0
      epsilon = 1e-8
      alpha_t = tf.scalar_mul(alpha, tf.ones_like(onehot_labels, dtype=tf.float32))
      alpha_t = tf.where(tf.equal(onehot_labels, 1.0), alpha_t, 1 - alpha_t)
      losses = tf.reduce_sum(
        -alpha_t * tf.pow(1. - predictions_pt, gamma) * onehot_labels * tf.log(predictions_pt + epsilon),
        name=name, axis=1)
      return losses
Exemplo n.º 6
0
    def build_graph(self,test_decoder_logits):
        print('starting building graph [sentiment-discriminator]')
        with tf.variable_scope("sentiment") as scope:
            self.inputs = tf.slice(test_decoder_logits,[0,0,0],[self.batch_size,self.max_length,self.vocab_size])
            # variable
            weights = {
                'w2v' : tf.get_variable(initializer = tf.random_uniform_initializer(-0.1, 0.1, dtype=tf.float32),shape = [self.vocab_size, self.embedding_dim], name='w2v'),
                'out_1' : tf.get_variable(initializer = tf.random_normal_initializer(), shape = [self.unit_size*2, 1], name='w_out_1'),
            }
            biases = {
            'out_1' : tf.get_variable(initializer = tf.random_normal_initializer(), shape=[1], name='b_out_1'),
            }
            # structure
            def BiRNN(x):
                x = tf.unstack(x, self.max_length, 1)
                lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(self.unit_size, forget_bias=1.0)
                lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(self.unit_size,forget_bias=1.0)
                outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32 )
                return outputs[-1]

            self.inputs_softmax = tf.nn.softmax(tf.scalar_mul(tf.constant(5.0, shape=[]),self.inputs))
            y_list=[]
            for i in range(self.inputs.get_shape().as_list()[0]):
                y = tf.matmul(self.inputs_softmax[i], weights['w2v'])
                y = tf.reshape(y, [1, self.max_length, self.embedding_dim])
                y_list.append(y)
            embbed_layer = tf.concat(y_list,0)
            layer_1 = BiRNN(embbed_layer)
            pred = tf.matmul(layer_1, weights['out_1']) + biases['out_1'] 
            # get score
            self.score = tf.sigmoid(pred)
Exemplo n.º 7
0
 def test_decoder_loop(prev,i):
     factor = tf.constant(5,shape=(),dtype=tf.float32)
     prev = tf.scalar_mul(factor,tf.add(tf.matmul(prev,weight_output),bias_output))
     prev_index = tf.nn.softmax(prev) 
     pred_prev = tf.matmul(prev_index,word_embedding_matrix)
     next_input = pred_prev
     return next_input
Exemplo n.º 8
0
    def init_training_graph(self):

        with tf.name_scope('Evaluation'):
            logits = self.last
            prob_b = tf.squeeze(logits, squeeze_dims=[1,2])
            self.predictions = tf.argmax(prob_b, axis=1)
            
            with tf.name_scope('Loss'):
                
                self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prob_b,
                                                                          labels=tf.cast(self.train_labels_node, tf.int32),
                                                                          name="entropy")))
                tf.summary.scalar("entropy", self.loss)

            with tf.name_scope('Accuracy'):

                LabelInt = tf.cast(self.train_labels_node, tf.int64)
                CorrectPrediction = tf.equal(self.predictions, LabelInt)
                self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32))
                tf.summary.scalar("accuracy", self.accuracy)

            with tf.name_scope('Prediction'):

                self.TP = tf.count_nonzero(self.predictions * LabelInt)
                self.TN = tf.count_nonzero((self.predictions - 1) * (LabelInt - 1))
                self.FP = tf.count_nonzero(self.predictions * (LabelInt - 1))
                self.FN = tf.count_nonzero((self.predictions - 1) * LabelInt)

            with tf.name_scope('Precision'):

                self.precision = tf.divide(self.TP, tf.add(self.TP, self.FP))
                tf.summary.scalar('Precision', self.precision)

            with tf.name_scope('Recall'):

                self.recall = tf.divide(self.TP, tf.add(self.TP, self.FN))
                tf.summary.scalar('Recall', self.recall)

            with tf.name_scope('F1'):

                num = tf.multiply(self.precision, self.recall)
                dem = tf.add(self.precision, self.recall)
                self.F1 = tf.scalar_mul(2, tf.divide(num, dem))
                tf.summary.scalar('F1', self.F1)

            with tf.name_scope('MeanAccuracy'):
                
                Nprecision = tf.divide(self.TN, tf.add(self.TN, self.FN))
                self.MeanAcc = tf.divide(tf.add(self.precision, Nprecision) ,2)

            #self.batch = tf.Variable(0, name = "batch_iterator")

            self.train_prediction = tf.nn.softmax(logits)

            self.test_prediction = tf.nn.softmax(logits)

        tf.global_variables_initializer().run()

        print('Computational graph initialised')
def create_left_right(c, w, nrm): 
  left = tf.transpose(tf.mul(tf.transpose(nrm), w));
  right= tf.scalar_mul(-1.0, left);
  
  left = tf.add(left, c);
  right= tf.add(right,c);
  
  return left,right
Exemplo n.º 10
0
def main(_):
  print('loading word embeddings from %s' % FLAGS.embedding_file)
  weight_matrix, word_idx = sentiment.load_embeddings(FLAGS.embedding_file)

  train_file = os.path.join(FLAGS.tree_dir, 'train.txt')
  print('loading training trees from %s' % train_file)
  train_trees = sentiment.load_trees(train_file)

  dev_file = os.path.join(FLAGS.tree_dir, 'dev.txt')
  print('loading dev trees from %s' % dev_file)
  dev_trees = sentiment.load_trees(dev_file)

  with tf.Session() as sess:
    print('creating the model')
    keep_prob = tf.placeholder_with_default(1.0, [])
    train_feed_dict = {keep_prob: FLAGS.keep_prob}
    word_embedding = sentiment.create_embedding(weight_matrix)
    compiler, metrics = sentiment.create_model(
        word_embedding, word_idx, FLAGS.lstm_num_units, keep_prob)
    loss = tf.reduce_sum(compiler.metric_tensors['all_loss'])
    opt = tf.train.AdagradOptimizer(FLAGS.learning_rate)
    grads_and_vars = opt.compute_gradients(loss)
    found = 0
    for i, (grad, var) in enumerate(grads_and_vars):
      if var == word_embedding.weights:
        found += 1
        grad = tf.scalar_mul(FLAGS.embedding_learning_rate_factor, grad)
        grads_and_vars[i] = (grad, var)
    assert found == 1  # internal consistency check
    train = opt.apply_gradients(grads_and_vars)
    saver = tf.train.Saver()

    print('initializing tensorflow')
    sess.run(tf.global_variables_initializer())

    with compiler.multiprocessing_pool():
      print('training the model')
      train_set = compiler.build_loom_inputs(train_trees)
      dev_feed_dict = compiler.build_feed_dict(dev_trees)
      dev_hits_best = 0.0
      for epoch, shuffled in enumerate(td.epochs(train_set, FLAGS.epochs), 1):
        train_loss = 0.0
        for batch in td.group_by_batches(shuffled, FLAGS.batch_size):
          train_feed_dict[compiler.loom_input_tensor] = batch
          _, batch_loss = sess.run([train, loss], train_feed_dict)
          train_loss += batch_loss
        dev_metrics = sess.run(metrics, dev_feed_dict)
        dev_loss = dev_metrics['all_loss']
        dev_accuracy = ['%s: %.2f' % (k, v * 100) for k, v in
                        sorted(dev_metrics.items()) if k.endswith('hits')]
        print('epoch:%4d, train_loss: %.3e, dev_loss: %.3e, dev_accuracy: [%s]'
              % (epoch, train_loss, dev_loss, ' '.join(dev_accuracy)))
        dev_hits = dev_metrics['root_hits']
        if dev_hits > dev_hits_best:
          dev_hits_best = dev_hits
          save_path = saver.save(sess, FLAGS.checkpoint_base, global_step=epoch)
          print('model saved in file: %s' % save_path)
Exemplo n.º 11
0
  def _add_focal_losses(self, sigma_rpn=3.0):
    with tf.variable_scope('loss_' + self._tag) as scope:
      # RPN, class loss
      rpn_cls_score = tf.reshape(self._predictions['rpn_cls_score_reshape'], [-1, 2])
      rpn_label = tf.reshape(self._anchor_targets['rpn_labels'], [-1])
      rpn_select = tf.where(tf.not_equal(rpn_label, -1))
      rpn_cls_score = tf.reshape(tf.gather(rpn_cls_score, rpn_select), [-1, 2])
      rpn_label = tf.reshape(tf.gather(rpn_label, rpn_select), [-1])
      rpn_cross_entropy = tf.reduce_mean(
        tf.nn.sparse_softmax_cross_entropy_with_logits(logits=rpn_cls_score, labels=rpn_label))

      # RPN, bbox loss
      rpn_bbox_pred = self._predictions['rpn_bbox_pred']
      rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets']
      rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights']
      rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights']

      rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights,
                                          rpn_bbox_outside_weights, sigma=sigma_rpn, dim=[1, 2, 3])

      # RCNN, class loss
      alpha_scale = 0.25
      gamma = 2
      epsilon = 1e-8
      cls_score = self._predictions["cls_score"]
      label = tf.reshape(self._proposal_targets["labels"], [-1])
      label = tf.one_hot(label, depth=self._num_classes)
      cls_pred = tf.nn.sigmoid(cls_score)
      predictions_pt = tf.where(tf.equal(label, 1), cls_pred, 1-cls_pred)
      alpha_t = tf.ones_like(label, dtype=tf.float32)
      alpha_t = tf.scalar_mul(alpha_scale, alpha_t)
      alpha_t = tf.where(tf.equal(label, 1.0), alpha_t, 1. - alpha_t)
      cross_entropy = tf.reduce_mean(-alpha_t*tf.pow(1 - predictions_pt, gamma)*tf.log(predictions_pt + epsilon))

      # RCNN, bbox loss
      bbox_pred = self._predictions['bbox_pred']
      bbox_targets = self._proposal_targets['bbox_targets']
      bbox_inside_weights = self._proposal_targets['bbox_inside_weights']
      bbox_outside_weights = self._proposal_targets['bbox_outside_weights']

      loss_box = self._smooth_l1_loss(bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights)

      self._losses['cross_entropy'] = cross_entropy
      self._losses['loss_box'] = loss_box
      self._losses['rpn_cross_entropy'] = rpn_cross_entropy
      self._losses['rpn_loss_box'] = rpn_loss_box

      self._losses['rpn_loss'] = rpn_loss_box + rpn_cross_entropy
      self._losses['class_loss'] = cross_entropy + loss_box
      loss = cross_entropy + loss_box + rpn_cross_entropy + rpn_loss_box
      self._losses['total_loss'] = loss

      self._event_summaries.update(self._losses)

    return loss
def create_cost(center, xy, width, persitaltic, bend, bend_profiles, contour, contour_normals, length, weight_spacing = 1.0, weight_bending = 1.0, gamma = 1.0, kappa = 2.0):

  c = tf.add(center, xy);
  
  #tangent and normals  
  t  = create_tangent(c);
  tn = create_normalize_tangent(t);
  ta = create_average_tangent(tn);
  nl = create_normal(ta);

  #bend the center
  c = create_bend(center, nl, bend, bend_profiles);

  #peristaltically move the center
  c = create_peristaltic(c, ta, persitaltic);

  #tangents
  t  = create_tangent(c);
  tn = create_normalize_tangent(t);
  ta = create_average_tangent(tn);
  nl = create_normal(ta);
  nr = tf.scalar_mul(-1.0, nl);  
  l,r = create_left_right(c,width,nl);

  cost_left = create_cost_soft_min_aligned_distance(l, contour, nl, contour_normals, k = kappa, gamma = gamma);
  cost_right= create_cost_soft_min_aligned_distance(r, contour, nr, contour_normals, k = kappa, gamma = gamma);
  cost = tf.add(cost_left, cost_right);
  
  #spacing and bending
  if weight_spacing != 0:
    cost_spacing = tf.scalar_mul(weight_spacing, create_cost_spacing(t, length));
    cost = tf.add(cost, cost_spacing);
  else:
    cost_spacing = tf.constant(0);
  
  if weight_bending != 0:
    cost_bending = tf.scalar_mul(weight_bending, create_cost_bending(tn));
    cost = tf.add(cost, cost_bending);
  else:
    cost_bending = tf.constant(0);
  
  return (cost, cost_left, cost_right, cost_spacing, cost_bending, c, l, r, nl);  
Exemplo n.º 13
0
def focal_loss3(cls_score, label, num_classes):
    alpha_scale = 0.25
    gamma = 2
    epsilon = 1e-8
    label = tf.one_hot(label, depth=num_classes)
    cls_pred = tf.nn.sigmoid(cls_score)
    predictions_pt = tf.where(tf.equal(label, 1), cls_pred, 1 - cls_pred)
    alpha_t = tf.ones_like(label, dtype=tf.float32)
    alpha_t = tf.scalar_mul(alpha_scale, alpha_t)
    alpha_t = tf.where(tf.equal(label, 1.0), alpha_t, 1. - alpha_t)
    losses = tf.reduce_mean(-alpha_t * tf.pow(1 - predictions_pt, gamma) * tf.log(predictions_pt + epsilon), axis=1)
    return losses
Exemplo n.º 14
0
    def build_model(self):

        self.images = tf.placeholder(tf.float32, [self.batch_size] + [self.output_size, self.output_size, self.c_dim], name='real_images')
        self.sample_images= tf.placeholder(tf.float32, [self.sample_size] + [self.output_size, self.output_size, self.c_dim], name='sample_images')
        self.z = tf.placeholder(tf.float32, [None, self.z_dim], name='z')

        self.z_sum = tf.summary.histogram("z", self.z)

        self.G = self.generator(self.z)
        self.D = self.discriminator(self.images)
        self.D_ = self.discriminator(self.G, reuse=True)
        self.sampler = self.sampler(self.z)

        self.d_sum = tf.summary.histogram("d", self.D)
        self.d__sum = tf.summary.histogram("d_", self.D_)
        self.G_sum = tf.summary.image("G", self.G)

        self.d_loss_real = tf.reduce_mean(tf.scalar_mul(-1, self.D))
        self.d_loss_fake = tf.reduce_mean(self.D_)
        self.g_loss = tf.reduce_mean(tf.scalar_mul(-1, self.D_))

        self.d_loss_real_sum = tf.summary.scalar("d_loss_real", self.d_loss_real)
        self.d_loss_fake_sum = tf.summary.scalar("d_loss_fake", self.d_loss_fake)
                                                    
        self.d_loss = self.d_loss_real + self.d_loss_fake

        self.g_loss_sum = tf.summary.scalar("g_loss", self.g_loss)
        self.d_loss_sum = tf.summary.scalar("d_loss", self.d_loss)

        t_vars = tf.trainable_variables()

        self.d_vars = [var for var in t_vars if 'd_' in var.name]
        self.g_vars = [var for var in t_vars if 'g_' in var.name]

        self.epoch = tf.Variable(-1, name='epoch', trainable=False)
        self.increment_epoch = tf.assign(self.epoch, self.epoch+1)

        self.global_step = tf.Variable(0, name='global_step', trainable=False)
        self.saver = tf.train.Saver(max_to_keep=1000)
Exemplo n.º 15
0
def entropy_matrix(graph, P):
    """

    :param graph:
    :param P:
    :return:
    """
    with graph.as_default():
        shape = P.get_shape().as_list()
        one_diagonal = tf.diag(diagonal=[1.0 for _ in range(shape[0])])
        P_mod = tf.add(P, one_diagonal)
        H = tf.reduce_sum(tf.scalar_mul(-1.0, tf.mul(P, tf.log(P_mod))), 1)
    return H
Exemplo n.º 16
0
  def _compute_update(self, param, grad, state):
    """Compute updates of parameters."""

    # get the learning rate at the current index, if the index
    # is greater than the number of available learning rates,
    # use the last one
    index = tf.minimum(state["itr"], self.max_index)
    learning_rate = tf.gather(self.learning_rates, index)

    # update the parameters: parameter - learning_rate * gradient
    updated_param = param - tf.scalar_mul(learning_rate, grad)

    return updated_param, {"itr": state["itr"] + 1}
Exemplo n.º 17
0
    def init_training_graph(self):
        with tf.name_scope('Evaluation'):
            self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/")
            self.predictions = tf.argmax(self.logits, axis=3)
            
            with tf.name_scope('Loss'):
                self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,
                                                                          labels=tf.squeeze(tf.cast(self.train_labels_node, tf.int32), squeeze_dims=[3]),
                                                                          name="entropy")))
                tf.summary.scalar("entropy", self.loss)

            with tf.name_scope('Accuracy'):

                LabelInt = tf.squeeze(tf.cast(self.train_labels_node, tf.int64), squeeze_dims=[3])
                CorrectPrediction = tf.equal(self.predictions, LabelInt)
                self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32))
                tf.summary.scalar("accuracy", self.accuracy)

            with tf.name_scope('ClassPrediction'):
                flat_LabelInt = tf.reshape(LabelInt, [-1])
                flat_predictions = tf.reshape(self.predictions, [-1])
                self.cm = tf.confusion_matrix(flat_LabelInt, flat_predictions, self.NUM_LABELS)
                flatten_confusion_matrix = tf.reshape(self.cm, [-1])
                total = tf.reduce_sum(self.cm)
                for i in range(self.NUM_LABELS):
                    name = "Label_{}".format(i)
                    TP, TN, FP, FN = GetCMInfo_TF(self.cm, i, self.NUM_LABELS)

                    precision =  tf.divide(TP, tf.add(TP, FP))
                    recall = tf.divide(TP, tf.add(TP, FN))
                    num = tf.multiply(precision, recall)
                    dem = tf.add(precision, recall)
                    F1 = tf.scalar_mul(2, tf.divide(num, dem))
                    Nprecision = tf.divide(TN, tf.add(TN, FN))
                    MeanAcc = tf.divide(tf.add(precision, Nprecision) ,2)

                    tf.summary.scalar(name + '_Precision', precision)
                    tf.summary.scalar(name + '_Recall', recall)
                    tf.summary.scalar(name + '_F1', F1)
                    tf.summary.scalar(name + '_Performance', MeanAcc)
                confusion_image = tf.reshape( tf.cast( self.cm, tf.float32),
                                            [1, self.NUM_LABELS, self.NUM_LABELS, 1])
                tf.summary.image('confusion', confusion_image)

            self.train_prediction = tf.nn.softmax(self.logits)

            self.test_prediction = self.train_prediction

        tf.global_variables_initializer().run()

        print('Computational graph initialised')
def SimStep(x,In_Yr,In_Yp,In_k,In_dt): 
    Yr=tf.constant(In_Yr,tf.float32)# Reactant ratios for each reaction
    Yp=tf.constant(In_Yp,tf.float32)# Product ratios for each reaction
    k=tf.constant(In_k,tf.float32)# Reaction constants for each reaction
    dt=tf.constant(In_dt,tf.float32) # time lapse for each simulation step
    s1=tf.pow(x,Yr)
    s2=tf.reduce_prod(s1,1)
    r=k*s2#Reacion rates 
    s4=tf.scalar_mul(dt,r)
    Yd=Yp-Yr # Change in concentrations attribute to each reaction
    dxij=s4*tf.transpose(Yd)# concentration changes each reaction in this step***
    dx=tf.reduce_sum(dxij,1) #sum of concentration changes from all reactions in this step***
    xp=x+dx#New concentration after steps  
    return(xp)
Exemplo n.º 19
0
    def __init__(self, x, weights: Weights, b: float, hp: Hyperparameters) -> None:
        self.x = tf.transpose(x)
        self.weights = weights
        self.b = b

        with tf.variable_scope('Learning'):
            self.alpha_ma_w = tf.scalar_mul(scalar=hp.alpha_ma, x=self.weights.w)
            self.one_minus_alpha_ma_x = tf.scalar_mul(scalar=1. - hp.alpha_ma, x=self.x)
            self.new_w = self.alpha_ma_w + self.one_minus_alpha_ma_x
            self.learn_weights_op = self.weights.assign(self.new_w)

        with tf.variable_scope('Layer'):
            with tf.variable_scope('Wx_plus_b'):
                z = tf.add(tf.matmul(x, weights.w), self.b)
            activation = tf.nn.relu(z, name='Activation')
            excitatory, inhibitory = weights.split(activation)

            self.excitatory_y = excitatory
            self.inhibitory_y = tf.scalar_mul(scalar=-1., x = inhibitory)

            self.y = tf.concat([self.excitatory_y, self.inhibitory_y], axis=1)
            util.variable_summaries(self.y)
            print(self.y, self.y.shape)
Exemplo n.º 20
0
def AddGaussianNoise(t, sigma, noise_rate, name=None):
  """Add i.i.d. Gaussian noise (0, sigma^2) to every entry of t.
  Args:
    t: the input tensor.
    sigma: the stddev of the Gaussian noise.
    name: optional name.
  Returns:
    the noisy tensor.
  """

  with tf.name_scope(values=[t, sigma], name=name,
                     default_name="add_gaussian_noise") as name:
    noisy_t = t + tf.scalar_mul(noise_rate, tf.random_normal(tf.shape(t), stddev=sigma))
  return noisy_t
Exemplo n.º 21
0
def probability_matrix(graph, beta, D):
    """
    :param D:
    :return:
    """
    with graph.as_default():
        shape = D.get_shape().as_list()
        beta_matrix = tf.tile(tf.expand_dims(beta, 1), multiples=[1, shape[1]])
        sqrt_matrix = tf.tile(tf.constant(0.5, dtype=tf.float32, shape=[1,1]), multiples=[shape[0], shape[1]])
        zero_diagonal = tf.exp(tf.diag(diagonal=[-np.inf for _ in range(shape[0])]))
        exp = tf.mul(tf.exp(tf.mul(beta_matrix, tf.scalar_mul(-1.0, tf.pow(D, sqrt_matrix)))), zero_diagonal)
        sum_exp = tf.reduce_sum(exp, reduction_indices=1)
        sum_exp_matrix = tf.tile(tf.expand_dims(sum_exp, 1), multiples=[1, shape[1]])
        return tf.div(exp, sum_exp_matrix)
Exemplo n.º 22
0
def calculate_euclidean_distance(training_data, test_data):
    # computing (x-y)^2 = x^2 + y^2 - 2xy
    first_term = tf.reduce_sum(training_data**2, axis=1)
    second_term = tf.expand_dims(tf.reduce_sum(test_data**2, axis=1), axis=1)
    xy = tf.matmul(test_data, tf.transpose(training_data))
    third_term = tf.scalar_mul(-2, xy)
    dist = first_term + second_term + third_term

    # shape1 = tf.shape(first_term)
    # shape2 = tf.shape(second_term)
    # shape3 = tf.shape(third_term)
    # sess = tf.InteractiveSession()
    # print(sess.run([dist], feed_dict={X: trainData, Z: testData}))
    # print(sess.run([shape1, shape2, shape3], feed_dict={X: trainData, Z: testData}))
    return dist
Exemplo n.º 23
0
def apply_weightshared_on_grads(grads_and_vars, dict_cenidx):
    # Mask gradients with pruned elements
    import config
    for key, cenidx in dict_cenidx.items():
        count = 0
        for grad, var in grads_and_vars:
                if var.name == key+":0":
                    num_cen = config.kmeans_para[key] 
                    tem_grad=[]    
                    for i in range(num_cen):
                        cenidx_obj = tf.cast(tf.constant(cenidx == i), tf.float32)
                        tem_grad.append(tf.scalar_mul(tf.reduce_sum(tf.multiply(cenidx_obj,grad)),cenidx_obj))
                    grad=(tf.add_n(tem_grad))
                    grads_and_vars[count] = (grad, var)
                count += 1
    return grads_and_vars
Exemplo n.º 24
0
def get_batch(filename, batch_size):
  if not tf.gfile.Exists(filename):
    raise ValueError('Failed to find file ' + filename)
  # get single examples
  label, image = _read_and_decode_single_example(filename)
  # groups examples into batches randomly
  images_batch, labels_batch = tf.train.shuffle_batch(
      [image, label], batch_size=batch_size,
      capacity=2000,
      min_after_dequeue=1000)

  images_batch = tf.cast(images_batch, tf.float32)
  norm_factor = tf.constant(1.0/255)
  images_batch = tf.scalar_mul(norm_factor, images_batch)
  #labels_batch = tf.cast(labels_batch, tf.float32)

  return images_batch, labels_batch
Exemplo n.º 25
0
def distance_matrix(graph, X):
    """
    :param X: Input Matrix for which to calculate Distance Matrix
    :return:
    """
    with graph.as_default():
        shape = X.get_shape().as_list()
        Di = tf.reduce_sum(tf.mul(X, X), 1)
        I_magnitude_tiled = tf.tile(tf.expand_dims(Di, 1), multiples=[1, shape[0]])
        I_t_magnitude_tiled = tf.tile(tf.expand_dims(Di, 0), multiples=[shape[0], 1])
        I_I = tf.matmul(X, tf.transpose(X))
        D = tf.add(
            tf.add(I_magnitude_tiled, I_t_magnitude_tiled),
            tf.scalar_mul(-2.0, I_I)
        )
        zero_diagonal = tf.exp(tf.diag(diagonal=[-np.inf for _ in range(shape[0])]))
    return tf.mul(D, zero_diagonal)
def create_cost_soft_min_distance(self, c, s):
  """Creates a soft-min distance of the centers to the points"""
  c_shape = c.get_shape().as_list();        
  s_shape = s.get_shape().as_list();
  
  #expand matrices
  cc = tf.reshape(c, [c_shape[0], c_shape[1], 1]);    
  ss = tf.reshape(s, [s_shape[0], s_shape[1], 1]);
  ss = tf.transpose(ss, perm = [0,2,1]);
  cc = tf.tile(cc, [1, 1, s_shape[0]]);
  ss = tf.tile(ss, [c_shape[0], 1, 1]);
  
  #pairwise distances
  dist2 = tf.sqrt(tf.reduce_sum(tf.squared_difference(cc,ss), reduction_indices = 1));
  dist2 = tf.reduce_mean(dist2, reduction_indices=0); # hack: get rid of batches here 
  
  #softmin
  return tf.reduce_sum(tf.mul(tf.nn.softmax(tf.scalar_mul(tf.constant(-1.0,"float32"), dist2)), dist2),reduction_indices = 0);
Exemplo n.º 27
0
def clip_norm(g, c, n):
    """Clip the gradient `g` if the L2 norm `n` exceeds `c`.

    # Arguments
        g: Tensor, the gradient tensor
        c: float >= 0. Gradients will be clipped
            when their L2 norm exceeds this value.
        n: Tensor, actual norm of `g`.

    # Returns
        Tensor, the gradient clipped if required.
    """
    if c <= 0:  # if clipnorm == 0 no need to add ops to the graph
        return g

    # tf require using a special op to multiply IndexedSliced by scalar
    if K.backend() == 'tensorflow':
        condition = n >= c
        then_expression = tf.scalar_mul(c / n, g)
        else_expression = g

        # saving the shape to avoid converting sparse tensor to dense
        if isinstance(then_expression, tf.Tensor):
            g_shape = copy.copy(then_expression.get_shape())
        elif isinstance(then_expression, tf.IndexedSlices):
            g_shape = copy.copy(then_expression.dense_shape)
        if condition.dtype != tf.bool:
            condition = tf.cast(condition, 'bool')
        g = tf.cond(condition,
                    lambda: then_expression,
                    lambda: else_expression)
        if isinstance(then_expression, tf.Tensor):
            g.set_shape(g_shape)
        elif isinstance(then_expression, tf.IndexedSlices):
            g._dense_shape = g_shape
    else:
        g = K.switch(K.greater_equal(n, c), g * c / n, g)
    return g
def clip_norm(g, c, n):
    if c > 0:
        if K.backend() == 'tensorflow':
            import tensorflow as tf
            import copy
            condition = n >= c
            then_expression = tf.scalar_mul(c / n, g)
            else_expression = g

            if hasattr(then_expression, 'get_shape'):
                g_shape = copy.copy(then_expression.get_shape())
            elif hasattr(then_expression, 'dense_shape'):
                g_shape = copy.copy(then_expression.dense_shape)
            if condition.dtype != tf.bool:
                condition = tf.cast(condition, 'bool')
            g = K.tensorflow_backend.control_flow_ops.cond(
                condition, lambda: then_expression, lambda: else_expression)
            if hasattr(then_expression, 'get_shape'):
                g.set_shape(g_shape)
            elif hasattr(then_expression, 'dense_shape'):
                g._dense_shape = g_shape
        else:
            g = K.switch(n >= c, g * c / n, g)
    return g
Exemplo n.º 29
0
    def train_setup(self):
        tf.set_random_seed(self.conf.random_seed)

        # Create queue coordinator.
        self.coord = tf.train.Coordinator()

        # Input size
        input_size = (self.conf.input_height, self.conf.input_width)

        # Load reader
        with tf.name_scope("create_inputs"):
            reader = ImageReader(self.conf.data_dir, self.conf.data_list,
                                 input_size, self.conf.random_scale,
                                 self.conf.random_mirror,
                                 self.conf.ignore_label, IMG_MEAN, self.coord)
            self.image_batch, self.label_batch = reader.dequeue(
                self.conf.batch_size)

        # Create network
        if self.conf.encoder_name not in ['res101', 'res50', 'deeplab']:
            print('encoder_name ERROR!')
            print("Please input: res101, res50, or deeplab")
            sys.exit(-1)
        elif self.conf.encoder_name == 'deeplab':
            net = Deeplab_v2(self.image_batch, self.conf.num_classes, True)
            # Variables that load from pre-trained model.
            restore_var = [
                v for v in tf.global_variables() if 'fc' not in v.name
            ]
            # Trainable Variables
            all_trainable = tf.trainable_variables()
            # Fine-tune part
            encoder_trainable = [
                v for v in all_trainable if 'fc' not in v.name
            ]  # lr * 1.0
            # Decoder part
            decoder_trainable = [v for v in all_trainable if 'fc' in v.name]
        else:
            net = ResNet_segmentation(self.image_batch, self.conf.num_classes,
                                      True, self.conf.encoder_name)
            # Variables that load from pre-trained model.
            restore_var = [
                v for v in tf.global_variables() if 'resnet_v1' in v.name
            ]
            # Trainable Variables
            all_trainable = tf.trainable_variables()
            # Fine-tune part
            encoder_trainable = [
                v for v in all_trainable if 'resnet_v1' in v.name
            ]  # lr * 1.0
            # Decoder part
            decoder_trainable = [
                v for v in all_trainable if 'decoder' in v.name
            ]

        decoder_w_trainable = [
            v for v in decoder_trainable
            if 'weights' in v.name or 'gamma' in v.name
        ]  # lr * 10.0
        decoder_b_trainable = [
            v for v in decoder_trainable
            if 'biases' in v.name or 'beta' in v.name
        ]  # lr * 20.0
        # Check
        assert (len(all_trainable) == len(decoder_trainable) +
                len(encoder_trainable))
        assert (len(decoder_trainable) == len(decoder_w_trainable) +
                len(decoder_b_trainable))

        # Network raw output
        raw_output = net.outputs  # [batch_size, h, w, 21]

        # Output size
        output_shape = tf.shape(raw_output)
        output_size = (output_shape[1], output_shape[2])

        # Groud Truth: ignoring all labels greater or equal than n_classes
        label_proc = prepare_label(self.label_batch,
                                   output_size,
                                   num_classes=self.conf.num_classes,
                                   one_hot=False)
        raw_gt = tf.reshape(label_proc, [
            -1,
        ])
        indices = tf.squeeze(
            tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)), 1)
        gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
        raw_prediction = tf.reshape(raw_output, [-1, self.conf.num_classes])
        prediction = tf.gather(raw_prediction, indices)

        # Pixel-wise softmax_cross_entropy loss
        loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=prediction, labels=gt)
        # L2 regularization
        l2_losses = [
            self.conf.weight_decay * tf.nn.l2_loss(v) for v in all_trainable
            if 'weights' in v.name
        ]
        # Loss function
        self.reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)

        # Define optimizers
        # 'poly' learning rate
        base_lr = tf.constant(self.conf.learning_rate)
        self.curr_step = tf.placeholder(dtype=tf.float32, shape=())
        learning_rate = tf.scalar_mul(
            base_lr,
            tf.pow((1 - self.curr_step / self.conf.num_steps),
                   self.conf.power))
        # We have several optimizers here in order to handle the different lr_mult
        # which is a kind of parameters in Caffe. This controls the actual lr for each
        # layer.
        opt_encoder = tf.train.MomentumOptimizer(learning_rate,
                                                 self.conf.momentum)
        opt_decoder_w = tf.train.MomentumOptimizer(learning_rate * 10.0,
                                                   self.conf.momentum)
        opt_decoder_b = tf.train.MomentumOptimizer(learning_rate * 20.0,
                                                   self.conf.momentum)
        # To make sure each layer gets updated by different lr's, we do not use 'minimize' here.
        # Instead, we separate the steps compute_grads+update_params.
        # Compute grads
        grads = tf.gradients(
            self.reduced_loss,
            encoder_trainable + decoder_w_trainable + decoder_b_trainable)
        grads_encoder = grads[:len(encoder_trainable)]
        grads_decoder_w = grads[len(encoder_trainable):(
            len(encoder_trainable) + len(decoder_w_trainable))]
        grads_decoder_b = grads[(len(encoder_trainable) +
                                 len(decoder_w_trainable)):]
        # Update params
        train_op_conv = opt_encoder.apply_gradients(
            zip(grads_encoder, encoder_trainable))
        train_op_fc_w = opt_decoder_w.apply_gradients(
            zip(grads_decoder_w, decoder_w_trainable))
        train_op_fc_b = opt_decoder_b.apply_gradients(
            zip(grads_decoder_b, decoder_b_trainable))
        # Finally, get the train_op!
        update_ops = tf.get_collection(
            tf.GraphKeys.UPDATE_OPS
        )  # for collecting moving_mean and moving_variance
        with tf.control_dependencies(update_ops):
            self.train_op = tf.group(train_op_conv, train_op_fc_w,
                                     train_op_fc_b)

        # Saver for storing checkpoints of the model
        self.saver = tf.train.Saver(var_list=tf.global_variables(),
                                    max_to_keep=0)

        # Loader for loading the pre-trained model
        self.loader = tf.train.Saver(var_list=restore_var)

        # Training summary
        # Processed predictions: for visualisation.
        raw_output_up = tf.image.resize_bilinear(raw_output, input_size)
        raw_output_up = tf.argmax(raw_output_up, axis=3)
        self.pred = tf.expand_dims(raw_output_up, dim=3)
        # Image summary.
        images_summary = tf.py_func(inv_preprocess,
                                    [self.image_batch, 2, IMG_MEAN], tf.uint8)
        labels_summary = tf.py_func(
            decode_labels, [self.label_batch, 2, self.conf.num_classes],
            tf.uint8)
        preds_summary = tf.py_func(decode_labels,
                                   [self.pred, 2, self.conf.num_classes],
                                   tf.uint8)
        self.total_summary = tf.summary.image(
            'images',
            tf.concat(axis=2,
                      values=[images_summary, labels_summary, preds_summary]),
            max_outputs=2)  # Concatenate row-wise.
        if not os.path.exists(self.conf.logdir):
            os.makedirs(self.conf.logdir)
        self.summary_writer = tf.summary.FileWriter(
            self.conf.logdir, graph=tf.get_default_graph())
    def timeflow(self, t=0.0):

        wbs = tf.matmul(self.body.wb, self.body.Q)  #[1,3] matrix
        tot_lbtomots = []

        Feqc = tf.constant([0, 0, -Mtot * 9.81], dtype=tf.float32)
        Teqc = tf.zeros([1, 3], dtype=tf.float32)

        #List of External Forces
        Flist = []
        llist = []
        for p in range(numLeg):
            for i in range(numsubleg):
                #print('alpha = ',self.leg[p].sub[i].alpha);
                self.leg[p].sub[i].omega += self.leg[p].sub[
                    i].alpha * dtime  #omega를 시간에 따라 갱신
                #print('omega = ',self.leg[p].sub[i].omega);
                self.leg[p].sub[i].theta += self.leg[p].sub[
                    i].omega * dtime  #theta를 시간에 따라 갱신
                #print('theta = ',self.leg[p].sub[i].theta);
                self.leg[p].sub[i].Q = tf.scalar_mul(tf.cos(self.leg[p].sub[i].theta), tf.eye(3, dtype=tf.float32)) + \
                tf.scalar_mul(1.-tf.cos(self.leg[p].sub[i].theta), tf.matmul(self.leg[p].sub[i].axis, self.leg[p].sub[i].axis, transpose_a = True)) + \
                tf.scalar_mul(tf.sin(self.leg[p].sub[i].theta), tf.cross(tf.tile(self.leg[p].sub[i].axis,[3,1]), tf.eye(3, dtype=tf.float32)))

            Qs = [tf.matmul(self.leg[p].sub[0].Q,
                            self.body.Q)]  #Qs는 i번째 subleg에서 space로의 좌표변환
            #List of rotation matrices of each sublegs in space frame
            #Type : list of [3,3] Tensor
            for i in range(1, numsubleg):
                Qs.append(tf.matmul(self.leg[p].sub[i].Q, Qs[i - 1]))

            e = [
                tf.matmul(self.leg[p].sub[i].axis, Qs[i])
                for i in range(numsubleg)
            ]
            #List of axes of each sublegs in space frame
            #Type : list of [None,3] Tensor

            Qw = [
                tf.scalar_mul(self.leg[p].sub[i].omega, e[i])
                for i in range(numsubleg)
            ]

            ws = [wbs + Qw[0]]
            for i in range(1, numsubleg):
                ws.append(ws[i - 1] + Qw[i])

            ls = [[
                tf.matmul(self.leg[p].sub[i].l[0], Qs[i]),
                tf.matmul(self.leg[p].sub[i].l[1], Qs[i])
            ] for i in range(numsubleg)]  #ls = 2Dtensor

            lbtomotbs = tf.matmul(self.body.lbtomot[p],
                                  self.body.Q)  # lbtomotbs = 2Dtensor

            lbtomots = [lbtomotbs + ls[0][0]]  # lbtomots = 2Dtensor

            for i in range(1, numsubleg):
                lbtomots.append(lbtomots[i - 1] + ls[i - 1][1] + ls[i][0])

            #Calculating External Forces
            vstmp = self.body.vs + tf.cross(wbs, lbtomotbs)
            NormalScale = 300.0
            TanhConst = 100.0

            Zfilter = tf.constant([[0., 0., 1.]])
            negZfilter = tf.constant([[0., 0., -1.]])
            XYfilter = tf.constant([[1., 1., 0.]])
            for i in range(numsubleg):
                Fz_primi = tf.multiply(negZfilter,
                                       self.body.rs + lbtomots[i] + ls[i][1])

                colbool = tf.reshape(
                    tf.matmul(Zfilter, tf.nn.relu(Fz_primi), transpose_b=True),
                    [])

                Fz = tf.scalar_mul(NormalScale, tf.nn.relu(Fz_primi))
                vstmp += tf.cross(ws[i], ls[i][0] + ls[i][1])
                vstmp_z = tf.reshape(
                    tf.matmul(Zfilter, vstmp, transpose_b=True), [])

                Vscale = 3. - tf.nn.tanh(TanhConst * vstmp_z)

                Fnormal = tf.scalar_mul(Vscale, Fz)

                Flist.append(Fnormal)

                vstmp_xy = tf.multiply(XYfilter, vstmp)
                Ffric = tf.scalar_mul(-Fricscale,
                                      tf.scalar_mul(colbool, vstmp_xy))

                Flist.append(Ffric)

                Feqc += Fnormal
                Feqc += Ffric

                Teqc += tf.cross(lbtomots[i] + ls[i][1], Fnormal + Ffric)
                llist.append(tf.norm(lbtomots[i] + ls[i][1]))
                #Teqc+=

            tot_lbtomots += lbtomots
        asb = tf.scalar_mul(Mtotinv, Feqc)
        alphab = tf.matmul(tf.matmul(Teqc, self.body.Q, transpose_b=True),
                           Ibinv)
        self.body.wb += tf.scalar_mul(dtime, alphab)
        self.body.Q += tf.scalar_mul(
            dtime, tf.cross(tf.concat([wbs, wbs, wbs], axis=0), self.body.Q))
        self.body.vs += tf.scalar_mul(dtime, asb)
        self.body.rs += tf.scalar_mul(dtime, self.body.vs)
        # Q to quaternion
        '''
        qw = tf.scalar_mul(0.5, tf.sqrt(tf.reduce_sum(tf.diag_part(self.body.Q))+1.))
        qv = tf.reduce_sum(tf.cross(self.body.Q, tf.eye(3, dtype = tf.float32)), axis = 0)/tf.scalar_mul(4., qw)

        # quaternion normalization
        
        qvsquare = tf.reduce_sum(tf.square(qv))
        qnorm = tf.sqrt(tf.square(qw)+qvsquare)
        qw /= qnorm
        qv /= qnorm
        # quaternion to Q

        self.body.Q = tf.scalar_mul(qw*qw-qvsquare,tf.eye(3, dtype = tf.float32))\
            + 2 * tf.matmul(tf.reshape(qv, [3, 1]), tf.reshape(qv, [1, 3]))\
            - 2 * qw * tf.cross(tf.tile(tf.reshape(qv, [1,3]), [3,1]), tf.eye(3, dtype = tf.float32))
        '''
        return llist, Flist, asb, Qs, [self.body.rs] + [
            x + self.body.rs for x in tot_lbtomots
        ]  #, qnorm, qw
Exemplo n.º 31
0
def train():
    """Train the Vnet model"""
    with tf.Graph().as_default():
        global_step = tf.train.get_or_create_global_step()

        # patch_shape(batch_size, height, width, depth, channels)
        input_batch_shape = (FLAGS.batch_size, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1) 
        output_batch_shape = (FLAGS.batch_size, FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer, 1) 
        
        images_placeholder, labels_placeholder = placeholder_inputs(input_batch_shape,output_batch_shape)

        for batch in range(FLAGS.batch_size):
            images_log = tf.cast(images_placeholder[batch:batch+1,:,:,:,0], dtype=tf.uint8)
            labels_log = tf.cast(tf.scalar_mul(255,labels_placeholder[batch:batch+1,:,:,:,0]), dtype=tf.uint8)

            tf.summary.image("image", tf.transpose(images_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
            tf.summary.image("label", tf.transpose(labels_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
        '''
        import glob
        imagelist = glob.glob(os.path.join(FLAGS.data_dir,'Volume','*.nii.gz'))
        labellist = [imagename.replace('Volume','Label').replace('volume','label') for imagename in imagelist]
        imagetrainlist = imagelist[:int(len(imagelist)/2)]
        labeltrainlist = labellist[:int(len(imagelist)/2)]
        imagetestlist  = imagelist[int(len(imagelist)/2):]
        labeltestlist  = labellist[int(len(imagelist)/2):]
        
        '''
        # Get images and labels
        train_data_dir = os.path.join(FLAGS.data_dir,'training')
        test_data_dir = os.path.join(FLAGS.data_dir,'testing')
        # support multiple image input, but here only use single channel, label file should be a single file with different classes
        image_filename = 'image_windowed.nii'
        label_filename = 'label.nii'
        # image_filename = 'img.nii.gz'
        # label_filename = 'label.nii.gz'

        # Force input pipepline to CPU:0 to avoid operations sometimes ended up at GPU and resulting a slow down
        with tf.device('/cpu:0'):
            # create transformations to image and labels
            trainTransforms = [
                NiftiDataset.Normalization(),
                NiftiDataset.Resample(0.2),
                NiftiDataset.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
                NiftiDataset.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel),
                NiftiDataset.RandomNoise()
                ]

            TrainDataset = NiftiDataset.NiftiDataset(
                data_dir=train_data_dir,
                image_filename=image_filename,
                label_filename=label_filename,
                transforms=trainTransforms,
                train=True
                )
            
            trainDataset = TrainDataset.get_dataset()
            trainDataset = trainDataset.shuffle(buffer_size=5)
            trainDataset = trainDataset.batch(FLAGS.batch_size)

            testTransforms = [
                NiftiDataset.Normalization(),
                NiftiDataset.Resample(0.2),
                NiftiDataset.Padding((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer)),
                NiftiDataset.RandomCrop((FLAGS.patch_size, FLAGS.patch_size, FLAGS.patch_layer),FLAGS.drop_ratio,FLAGS.min_pixel)
                ]

            TestDataset = NiftiDataset.NiftiDataset(
                data_dir=test_data_dir,
                image_filename=image_filename,
                label_filename=label_filename,
                transforms=testTransforms,
                train=True
            )

            testDataset = TestDataset.get_dataset()
            testDataset = testDataset.shuffle(buffer_size=5)
            testDataset = testDataset.batch(FLAGS.batch_size)
            
        train_iterator = trainDataset.make_initializable_iterator()
        next_element_train = train_iterator.get_next()

        test_iterator = testDataset.make_initializable_iterator()
        next_element_test = test_iterator.get_next()

        # Initialize the model
        with tf.name_scope("vnet"):
            logits = VNet.v_net(images_placeholder,input_channels = input_batch_shape[4], output_channels =2)

        # # apply weight to the logic funciton
        # if FLAGS.class_weight <0 or FLAGS.class_weight>1:
        #     print("Class weight should between 0 and 1")
        #     exit()

        # if logits.shape[4] == 2:
        #     class_weight = tf.constant([FLAGS.class_weight,1.0-FLAGS.class_weight]) 
        #     weighted_logits = tf.multiply(logits,class_weight)
        # else:
        #     weighted_logits = logits

        for batch in range(FLAGS.batch_size):
            logits_log_0 = tf.cast(logits[batch:batch+1,:,:,:,0], dtype=tf.uint8)
            logits_log_1 = tf.cast(logits[batch:batch+1,:,:,:,1], dtype=tf.uint8)
            tf.summary.image("logits_0", tf.transpose(logits_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
            tf.summary.image("logits_1", tf.transpose(logits_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer)

        # # Exponential decay learning rate
        # train_batches_per_epoch = math.ceil(TrainDataset.data_size/FLAGS.batch_size)
        # decay_steps = train_batches_per_epoch*FLAGS.decay_steps

        with tf.name_scope("learning_rate"):
            learning_rate = FLAGS.init_learning_rate
        #     learning_rate = tf.train.exponential_decay(FLAGS.init_learning_rate,
        #         global_step,
        #         decay_steps,
        #         FLAGS.decay_factor,
        #         staircase=True)
        tf.summary.scalar('learning_rate', learning_rate)

        # softmax op for probability layer
        with tf.name_scope("softmax"):
            softmax_op = tf.nn.softmax(logits,name="softmax")

        for batch in range(FLAGS.batch_size):
            softmax_log_0 = tf.cast(tf.scalar_mul(255,softmax_op[batch:batch+1,:,:,:,0]), dtype=tf.uint8)
            softmax_log_1 = tf.cast(tf.scalar_mul(255,softmax_op[batch:batch+1,:,:,:,1]), dtype=tf.uint8)
            tf.summary.image("softmax_0", tf.transpose(softmax_log_0,[3,1,2,0]),max_outputs=FLAGS.patch_layer)
            tf.summary.image("softmax_1", tf.transpose(softmax_log_1,[3,1,2,0]),max_outputs=FLAGS.patch_layer)

        # Op for calculating loss
        with tf.name_scope("cross_entropy"):
            loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
                logits=logits,
                labels=tf.squeeze(labels_placeholder, 
                squeeze_dims=[4])))
            # loss_op = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(
            #     logits=logits,
            #     labels=tf.squeeze(labels_placeholder,squeeze_dims=[4]),
            #     weights=[[]]))
        tf.summary.scalar('loss',loss_op)

        # Argmax Op to generate label from logits
        with tf.name_scope("predicted_label"):
            pred = tf.argmax(logits, axis=4 , name="prediction")

        for batch in range(FLAGS.batch_size):
            pred_log = tf.cast(tf.scalar_mul(255,pred[batch:batch+1,:,:,:]), dtype=tf.uint8)
            tf.summary.image("pred", tf.transpose(pred_log,[3,1,2,0]),max_outputs=FLAGS.patch_layer)

        # Accuracy of model
        with tf.name_scope("accuracy"):
            correct_pred = tf.equal(tf.expand_dims(pred,-1), tf.cast(labels_placeholder,dtype=tf.int64))
            accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
        tf.summary.scalar('accuracy', accuracy)

        # Dice Similarity
        with tf.name_scope("dice"):
            sorensen = dice_coe(tf.expand_dims(pred,-1),tf.cast(labels_placeholder,dtype=tf.int64), loss_type='sorensen')
            jaccard = dice_coe(tf.expand_dims(pred,-1),tf.cast(labels_placeholder,dtype=tf.int64), loss_type='jaccard')
            sorensen_loss = 1. - sorensen
            jaccard_loss = 1. - jaccard
        tf.summary.scalar('sorensen', sorensen)
        tf.summary.scalar('jaccard', jaccard)
        tf.summary.scalar('sorensen_loss', sorensen_loss)
        tf.summary.scalar('jaccard_loss',jaccard_loss)

        # Training Op
        with tf.name_scope("training"):
            # optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
            optimizer = tf.train.GradientDescentOptimizer(learning_rate=FLAGS.init_learning_rate)
            train_op = optimizer.minimize(
                loss=loss_op,
                global_step=global_step)

        # # epoch checkpoint manipulation
        start_epoch = tf.get_variable("start_epoch", shape=[1], initializer= tf.zeros_initializer,dtype=tf.int32)
        start_epoch_inc = start_epoch.assign(start_epoch+1)

        # saver
        summary_op = tf.summary.merge_all()
        checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir ,"checkpoint")
        print("Setting up Saver...")
        saver = tf.train.Saver(keep_checkpoint_every_n_hours=5)

        # training cycle
        with tf.Session() as sess:
            # Initialize all variables
            sess.run(tf.global_variables_initializer())
            print("{}: Start training...".format(datetime.datetime.now()))

            # summary writer for tensorboard
            train_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
            test_summary_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)

            # restore from checkpoint
            if FLAGS.restore_training:
                # check if checkpoint exists
                if os.path.exists(checkpoint_prefix+"-latest"):
                    print("{}: Last checkpoint found at {}, loading...".format(datetime.datetime.now(),FLAGS.checkpoint_dir))
                    latest_checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir,latest_filename="checkpoint-latest")
                    saver.restore(sess, latest_checkpoint_path)
            
            print("{}: Last checkpoint epoch: {}".format(datetime.datetime.now(),start_epoch.eval()[0]))
            print("{}: Last checkpoint global step: {}".format(datetime.datetime.now(),tf.train.global_step(sess, global_step)))

            # loop over epochs
            for epoch in np.arange(start_epoch.eval(), FLAGS.epochs):
                # initialize iterator in each new epoch
                sess.run(train_iterator.initializer)
                sess.run(test_iterator.initializer)
                print("{}: Epoch {} starts".format(datetime.datetime.now(),epoch+1))

                # training phase
                '''
                reader = sitk.ImageFileReader()
                for imagepath, labelpath in zip(imagetrainlist,labeltrainlist):
                    
                    reader.SetFileName(imagepath)
                    return reader.Execute()
                '''

                while True:
                    try:
                        [image, label] = sess.run(next_element_train)

                        image = image[:,:,:,:,np.newaxis]
                        label = label[:,:,:,:,np.newaxis]
                        
                        train, train_loss, train_accuracy, train_jaccard, summary = sess.run(
                            [train_op, loss_op, accuracy, jaccard, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label})
                        print("train loss: %.4f, accuracy: %.4f, jaccard: %.4f" % (train_loss, train_accuracy, train_jaccard))
                        train_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))

                    except tf.errors.OutOfRangeError:
                        start_epoch_inc.op.run()
                        # print(start_epoch.eval())
                        # save the model at end of each epoch training
                        print("{}: Saving checkpoint of epoch {} at {}...".format(datetime.datetime.now(),epoch+1,FLAGS.checkpoint_dir))
                        saver.save(sess, checkpoint_prefix, 
                            global_step=tf.train.global_step(sess, global_step),
                            latest_filename="checkpoint-latest")
                        print("{}: Saving checkpoint succeed".format(datetime.datetime.now()))
                        break
                # testing phase
                print("{}: Training of epoch {} finishes, testing start".format(datetime.datetime.now(),epoch+1))
                testJaccard = []
                while True:
                    try:
                        [image, label] = sess.run(next_element_test)

                        image = image[:,:,:,:,np.newaxis]
                        label = label[:,:,:,:,np.newaxis]
                        test_loss, test_accuracy, test_jaccard, summary = sess.run(
                            [loss_op, accuracy, jaccard, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label})
                        print("test loss: %.4f, accuracy: %.4f, jaccard: %.4f" % (test_loss, test_accuracy, test_jaccard))
                        testJaccard.append(test_jaccard)

                        #loss, summary = sess.run([loss_op, summary_op], feed_dict={images_placeholder: image, labels_placeholder: label})
                        test_summary_writer.add_summary(summary, global_step=tf.train.global_step(sess, global_step))

                    except tf.errors.OutOfRangeError:
                        print("mean testing Jaccard: {}".format(np.array(testJaccard).mean()))
                        break

        # close tensorboard summary writer
        train_summary_writer.close()
        test_summary_writer.close()
Exemplo n.º 32
0
		def apply_bernoulli_dropout():
			if scale_during_training:
				return tf.nn.dropout(inference, keep_prob)
			else:
				return tf.scalar_mul(keep_prob,tf.nn.dropout(inference, keep_prob))
Exemplo n.º 33
0
def main():
    """Create the model and start the training."""
    args = get_arguments()
    
    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)

    if args.center_crop_size is None:
        center_crop_size = None
    else:
        hc, wc = map(int, args.center_crop_size.split(','))
        center_crop_size = (hc, wc)

    with tf.name_scope("create_inputs"):
        reader = ImageReader(
            DATA_DIR,
            DATA_LIST_PATH,
            input_size,
            center_crop_size,
            args.random_scale,
            args.random_mirror,
            args.ignore_label,
            IMG_MEAN)
        image_batch, label_batch = reader.dequeue(args.batch_size)

    net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale)

    sub4_recls, sub24_recls, sub124_recls = bn_common.extend_reclassifier(net)

    restore_var = tf.global_variables()
    all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma]
   
    loss_sub4 = create_loss(sub4_recls, label_batch, args)
    loss_sub24 = create_loss(sub24_recls, label_batch, args)
    loss_sub124 = create_loss(sub124_recls, label_batch, args)
    
    l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables()
                 if ('weights' in v.name) or ('kernel' in v.name)]
    
    reduced_loss = LAMBDA1 * loss_sub4 +  LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses)

    # print(tf.get_variable_scope().name)
    # print(','.join([v.__op.original_name_scope for v in l2_losses]))
    # print(','.join([v for v in tf.trainable_variables() if ('beta' in v.name or 'gamma' in v.name)]))
    # tf.summary.FileWriter('./summary', tf.get_default_graph())
    # exit(0)

    # Using Poly learning rate policy 
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
    
    # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
    if args.update_mean_var == False:
        update_ops = None
    else:
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    with tf.control_dependencies(update_ops):
        opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
        grads = tf.gradients(reduced_loss, all_trainable)
        train_op = opt_conv.apply_gradients(zip(grads, all_trainable))

    # Set up tf session and initialize variables. 
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    sess.run(tf.global_variables_initializer())
    
    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=99)

    ckpt = tf.train.get_checkpoint_state(args.snapshot_dir)
    if ckpt and ckpt.model_checkpoint_path:
        loader = tf.train.Saver(var_list=restore_var)
        load(loader, sess, ckpt.model_checkpoint_path)
    else:
        print('Restore from pre-trained model...')
        net.load(args.restore_from, sess)

    # Start queue threads.
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        
        feed_dict = {step_ph: step}
        if step % args.save_pred_every == 0:
            loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
            save(saver, sess, args.snapshot_dir, step)
        else:
            loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
        duration = time.time() - start_time
        print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration))
        
    coord.request_stop()
    coord.join(threads)

    sess.close()
Exemplo n.º 34
0
def main():
    """Create the model and start the training."""
    args = get_arguments()
    
    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)
    
    tf.set_random_seed(args.random_seed)
    
    # Create queue coordinator.
    coord = tf.train.Coordinator()
    
    # Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReader(
            args.data_dir,
            args.data_list,
            input_size,
            args.random_scale,
            args.random_mirror,
            args.ignore_label,
            IMG_MEAN,
            coord)
        image_batch, label_batch = reader.dequeue(args.batch_size)
    
    # Create network.
    net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
    # For a small batch size, it is better to keep 
    # the statistics of the BN layers (running means and variances)
    # frozen, and to not update the values provided by the pre-trained model. 
    # If is_training=True, the statistics will be updated during the training.
    # Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
    # if they are presented in var_list of the optimiser definition.

    # Predictions.
    raw_output = net.layers['fc_out']
    # Which variables to load. Running means and variances are not trainable,
    # thus all_variables() should be restored.
    restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
    all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
    fc_trainable = [v for v in all_trainable if 'fc' in v.name]
    conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
    fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
    fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
    assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable))
    assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
    
    
    # Predictions: ignoring all predictions with labels greater or equal than n_classes
    raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
    label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w]
    raw_gt = tf.reshape(label_proc, [-1,])
    indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
    gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
    prediction = tf.gather(raw_prediction, indices)
                                                  
                                                  
    # Pixel-wise softmax loss.
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
    l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
    reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
    
    # Processed predictions: for visualisation.
    raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
    raw_output_up = tf.argmax(raw_output_up, dimension=3)
    pred = tf.expand_dims(raw_output_up, dim=3)
    
    # Image summary.
    images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
    labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
    preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)
    
    total_summary = tf.summary.image('images', 
                                     tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]), 
                                     max_outputs=args.save_num_images) # Concatenate row-wise.
    summary_writer = tf.summary.FileWriter(args.snapshot_dir,
                                           graph=tf.get_default_graph())
   
    # Define loss and optimisation parameters.
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
    
    opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
    opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
    opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)

    grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
    grads_conv = grads[:len(conv_trainable)]
    grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))]
    grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]

    train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
    train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
    train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))

    train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
    
    
    # Set up tf session and initialize variables. 
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()
    
    sess.run(init)
    
    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)

    ckpt = tf.train.get_checkpoint_state(SNAPSHOT_DIR)

    if ckpt and ckpt.model_checkpoint_path:
        loader = tf.train.Saver(var_list=restore_var)
        load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
        load(loader, sess, ckpt.model_checkpoint_path)
    else:
        print('No checkpoint file found.')
        load_step = 0

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        feed_dict = { step_ph : step }
        
        if step % args.save_pred_every == 0:
            loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, total_summary, train_op], feed_dict=feed_dict)
            summary_writer.add_summary(summary, step)
            save(saver, sess, args.snapshot_dir, step)
        else:
            loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
        duration = time.time() - start_time
        print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
        
    coord.request_stop()
    coord.join(threads)
Exemplo n.º 35
0
    def define(self):
        config = self.config
        with tf.Graph().as_default() as graph:
            self.graph = graph
            weights, biases = self.make_weights()
            self.training = tf.placeholder(tf.bool)

            # tf Graph Input
            self.X = X = tf.placeholder(
                tf.float32, [None, config['height'], config['width']])

            if config['conv']:
                X1 = conv_net(X, config, weights['conv'], biases['conv'],
                              config['conv']['max_pool_factors'])
                logits = fully_connected(X1, config, weights['fc'],
                                         biases['fc'], self.training)
            else:
                logits = fully_connected(X, config, weights['fc'],
                                         biases['fc'], self.training)

            n_classes = config['n_classes']
            self.Y = Y = tf.placeholder(tf.int32, [None, config['n_outputs']])
            Yhot = tf.one_hot(Y, n_classes)
            logits = tf.reshape(logits, [-1, config['n_outputs'], n_classes])
            logit_loss = self.logit_loss(logits, Yhot)

            if config['l2_loss']:
                if config['conv']:
                    l2_loss = tf.add_n(
                        map(lambda weight: tf.nn.l2_loss(weight),
                            weights['conv']))
                else:
                    l2_loss = tf.constant(0.)
                fc_l2_loss = tf.add_n(
                    map(lambda weight: tf.nn.l2_loss(weight), weights['fc']))
                l2_loss = tf.scalar_mul(config['l2_loss']['beta'],
                                        tf.add(l2_loss, fc_l2_loss))
            else:
                l2_loss = tf.constant(0.)

            self.cost = cost = logit_loss  #tf.add(logit_loss, l2_loss)
            tf.summary.scalar('cost', cost)

            self.pred = pred = tf.argmax(logits, axis=2)

            correct = tf.cast(tf.equal(tf.cast(pred, tf.int32), Y), tf.float32)
            correct_sequence = tf.cast(
                tf.equal(tf.reduce_sum(correct, 1), config['n_outputs']),
                tf.float32)

            self.accuracy = accuracy = tf.reduce_mean(correct_sequence)
            self.element_accuracy = element_accuracy = tf.reduce_mean(correct)
            tf.summary.scalar('set_accuracy', accuracy)
            tf.summary.scalar('element_accuracy', element_accuracy)

            self.global_step = global_step = tf.placeholder(tf.int32)

            if config['learning_rate']['decay']:
                learning_rate = tf.train.exponential_decay(
                    config['learning_rate']['start'], global_step,
                    config['iterations'],
                    config['learning_rate']['coefficient'])
            else:
                learning_rate = tf.constant(config['learning_rate']['start'])
            tf.summary.scalar('learning_rate', learning_rate)

            if config['optimizer'] == 'Adam':
                self.optimizer = optimizer = tf.train.AdamOptimizer(
                    learning_rate).minimize(logit_loss)
            elif config['optimizer'] == 'GradientDescent':
                self.optimizer = optimizer = tf.train.GradientDescentOptimizer(
                    learning_rate).minimize(cost)

            # Initializing the variables
            self.init = tf.global_variables_initializer()

            self.summary = tf.summary.merge_all()
            dirname = 'output/%s_%s' % (
                config['exp_name'],
                datetime.datetime.now().strftime('%Y_%m_%d_%H.%M'))
            self.train_writer = tf.summary.FileWriter(dirname, graph)

            # save hype for later comparisons
            with open(dirname + '/hypes.json', 'w') as f:
                json.dump(config, f, indent=2)
    def compute_gradients(self, loss, var_list=None,
                        gate_gradients=Optimizer.GATE_OP,
                        aggregation_method=None,
                        colocate_gradients_with_ops=False,
                        grad_loss=None):
        """Compute gradients of `loss` for the variables in `var_list`.
        This is the first part of `minimize()`.  It returns a list
        of (gradient, variable) pairs where "gradient" is the gradient
        for "variable".  Note that "gradient" can be a `Tensor`, an
        `IndexedSlices`, or `None` if there is no gradient for the
        given variable.
        Args:
        loss: A Tensor containing the value to minimize or a callable taking
            no arguments which returns the value to minimize. When eager execution
            is enabled it must be a callable.
        var_list: Optional list or tuple of `tf.Variable` to update to minimize
            `loss`.  Defaults to the list of variables collected in the graph
            under the key `GraphKeys.TRAINABLE_VARIABLES`.
        gate_gradients: How to gate the computation of gradients.  Can be
            `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`.
        aggregation_method: Specifies the method used to combine gradient terms.
            Valid values are defined in the class `AggregationMethod`.
        colocate_gradients_with_ops: If True, try colocating gradients with
            the corresponding op.
        grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.
        Returns:
        A list of (gradient, variable) pairs. Variable is always present, but
        gradient can be `None`.
        Raises:
        TypeError: If `var_list` contains anything else than `Variable` objects.
        ValueError: If some arguments are invalid.
        RuntimeError: If called with eager execution enabled and `loss` is
            not callable.
        @compatibility(eager)
        When eager execution is enabled, `gate_gradients`, `aggregation_method`,
        and `colocate_gradients_with_ops` are ignored.
        @end_compatibility
        """
        if callable(loss):
            with backprop.GradientTape() as tape:
                if var_list is not None:
                    tape.watch(var_list)
                    loss_value = loss()

            if var_list is None:
                var_list = tape.watched_variables()
            # TODO(jhseu): Figure out why GradientTape's gradients don't require loss
            # to be executed.
            with ops.control_dependencies([loss_value]):
                grads = tape.gradient(loss_value, var_list, grad_loss)
            return list(zip(grads, var_list))

        # # Non-callable/Tensor loss case
        # if context.executing_eagerly():
        #     raise RuntimeError(
        #         "`loss` passed to Optimizer.compute_gradients should "
        #         "be a function when eager execution is enabled.")

        if gate_gradients not in [SPSA.GATE_NONE, SPSA.GATE_OP, SPSA.GATE_GRAPH]:
            raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, "
                            "Optimizer.GATE_OP, Optimizer.GATE_GRAPH.  Not %s" %
                            gate_gradients)
        self._assert_valid_dtypes([loss])
        if grad_loss is not None:
            self._assert_valid_dtypes([grad_loss])
        if var_list is None:
            var_list = (
                variables.trainable_variables() +
                ops.get_collection(ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))
        else:
            var_list = nest.flatten(var_list)
        # pylint: disable=protected-access
        var_list += ops.get_collection(ops.GraphKeys._STREAMING_MODEL_PORTS)
        # pylint: enable=protected-access
        processors = [_get_processor(v) for v in var_list]
        if not var_list:
            raise ValueError("No variables to optimize.")
        var_refs = [p.target() for p in processors]

        # print("var_refs:")
        # for vr in var_refs:
        #     print(vr)

        # ==================================================================================
        # grads = gradients.gradients(
        #     loss, var_refs, grad_ys=grad_loss,
        #     gate_gradients=(gate_gradients == SPSA.GATE_OP),
        #     aggregation_method=aggregation_method,
        #     colocate_gradients_with_ops=colocate_gradients_with_ops)
        
        # grads = [ tf.zeros(tf.shape(vrefs)) for vrefs in var_refs ]
        orig_graph_view = None
        trainable_vars = var_list
        # self.tvars = var_list
        self.tvars = [var.name.split(':')[0] for var in var_list]  # list of names of trainable variables
        self.global_step_tensor = tf.Variable(0, name='global_step', trainable=False)

        # Perturbations
        deltas = {}
        n_perturbations = {}
        p_perturbations = {}
        with tf.name_scope("Perturbator"):
            self.c_t = tf.div( self.c,  tf.pow(tf.add(tf.cast(self.global_step_tensor, tf.float32),
                                              tf.constant(1, dtype=tf.float32)), self.gamma), name = "SPSA_ct" )
            for var in trainable_vars:
                self.num_params += self._mul_dims(var.get_shape())
                var_name = var.name.split(':')[0]
                random = Bernoulli(tf.fill(var.get_shape(), 0.5), dtype=tf.float32)
                deltas[var] = tf.subtract( tf.constant(1, dtype=tf.float32),
                                    tf.scalar_mul(tf.constant(2, dtype=tf.float32),random.sample(1)[0]), name = "SPSA_delta" )
                c_t_delta = tf.scalar_mul( tf.reshape(self.c_t, []), deltas[var] )
                n_perturbations[var_name+'/read:0'] = tf.subtract( var, c_t_delta, name = "perturb_n" )
                p_perturbations[var_name+'/read:0'] = tf.add(var, c_t_delta, name = "perturb_p" )
        # print("{} parameters".format(self.num_params))

        # Evaluator
        with tf.name_scope("Evaluator"):
            orig_graph_view = ge.sgv(tf.get_default_graph())
            _, self.ninfo = self._clone_model(orig_graph_view, n_perturbations, 'N_Eval')
            _, self.pinfo = self._clone_model(orig_graph_view, p_perturbations, 'P_Eval')

        # Weight Updater
        optimizer_ops = []
        grads = []
        with tf.control_dependencies([loss]):
            with tf.name_scope('Updater'):
                a_t = self.a / (tf.pow(tf.add(tf.cast(self.global_step_tensor, tf.float32),
                                             tf.constant(1, dtype=tf.float32)), self.alpha))
                for var in trainable_vars:
                    l_pos = self.pinfo.transformed( loss )
                    l_neg = self.ninfo.transformed( loss )
                    ghat = (l_pos - l_neg) / (tf.constant(2, dtype=tf.float32) * self.c_t * deltas[var])
                    optimizer_ops.append(tf.assign_sub(var, a_t*ghat))
                    grads.append(ghat)
        print(tf.get_default_graph())

        print("grads")
        for g in grads:
            print(g)


        #===================================================================================
        if gate_gradients == SPSA.GATE_GRAPH:
            print("===================")
            grads = control_flow_ops.tuple(grads)
        grads_and_vars = list(zip(grads, var_list))
        self._assert_valid_dtypes(
            [v for g, v in grads_and_vars
            if g is not None and v.dtype != dtypes.resource])
        return grads_and_vars
Exemplo n.º 37
0
    def forward(self, reshaped_input):
        cluster_weights = tf.get_variable(
            "cluster_weights", [self.feature_size, self.cluster_size],
            initializer=tf.random_normal_initializer(
                stddev=1 / math.sqrt(self.feature_size)))

        covar_weights = tf.get_variable(
            "covar_weights", [self.feature_size, self.cluster_size],
            initializer=tf.random_normal_initializer(
                mean=1.0, stddev=1 / math.sqrt(self.feature_size)))

        covar_weights = tf.square(covar_weights)
        eps = tf.constant([1e-6])
        covar_weights = tf.add(covar_weights, eps)

        tf.summary.histogram("cluster_weights", cluster_weights)
        activation = tf.matmul(reshaped_input, cluster_weights)
        if self.add_batch_norm:
            activation = slim.batch_norm(activation,
                                         center=True,
                                         scale=True,
                                         is_training=self.is_training,
                                         scope="cluster_bn")
        else:
            cluster_biases = tf.get_variable(
                "cluster_biases", [self.cluster_size],
                initializer=tf.random_normal(stddev=1 /
                                             math.sqrt(self.feature_size)))
            tf.summary.histogram("cluster_biases", cluster_biases)
            activation += cluster_biases

        activation = tf.nn.softmax(activation)
        tf.summary.histogram("cluster_output", activation)

        activation = tf.reshape(activation,
                                [-1, self.max_frames, self.cluster_size])

        a_sum = tf.reduce_sum(activation, -2, keep_dims=True)

        if not FLAGS.fv_couple_weights:
            cluster_weights2 = tf.get_variable(
                "cluster_weights2", [1, self.feature_size, self.cluster_size],
                initializer=tf.random_normal_initializer(
                    stddev=1 / math.sqrt(self.feature_size)))
        else:
            cluster_weights2 = tf.scalar_mul(FLAGS.fv_coupling_factor,
                                             cluster_weights)

        a = tf.multiply(a_sum, cluster_weights2)

        activation = tf.transpose(activation, perm=[0, 2, 1])

        reshaped_input = tf.reshape(reshaped_input,
                                    [-1, self.max_frames, self.feature_size])
        fv1 = tf.matmul(activation, reshaped_input)

        fv1 = tf.transpose(fv1, perm=[0, 2, 1])

        # computing second order FV
        a2 = tf.multiply(a_sum, tf.square(cluster_weights2))

        b2 = tf.multiply(fv1, cluster_weights2)
        fv2 = tf.matmul(activation, tf.square(reshaped_input))

        fv2 = tf.transpose(fv2, perm=[0, 2, 1])
        fv2 = tf.add_n([a2, fv2, tf.scalar_mul(-2, b2)])

        fv2 = tf.divide(fv2, tf.square(covar_weights))
        fv2 = tf.subtract(fv2, a_sum)

        fv2 = tf.reshape(fv2, [-1, self.cluster_size * self.feature_size])

        fv2 = tf.nn.l2_normalize(fv2, 1)
        fv2 = tf.reshape(fv2, [-1, self.cluster_size * self.feature_size])
        fv2 = tf.nn.l2_normalize(fv2, 1)

        fv1 = tf.subtract(fv1, a)
        fv1 = tf.divide(fv1, covar_weights)

        fv1 = tf.nn.l2_normalize(fv1, 1)
        fv1 = tf.reshape(fv1, [-1, self.cluster_size * self.feature_size])
        fv1 = tf.nn.l2_normalize(fv1, 1)

        return tf.concat([fv1, fv2], 1)
Exemplo n.º 38
0
def main(parsed_arguments):
    tf.enable_eager_execution()

    if not isinstance(parsed_arguments.threads, int):
        parsed_arguments.threads = int(parsed_arguments.threads)

    try:
        architecture_json_filename = parsed_arguments.json_filename
        architecture_json_content = open(architecture_json_filename,
                                         'r').read()
        parsed_architecture_json = json.loads(architecture_json_content)
    except:
        print('Expected a valid architecture json file.')

    assert os.path.isdir(parsed_arguments.input)

    if not isinstance(parsed_arguments.tile_size, int):
        parsed_arguments.tile_size = int(parsed_arguments.tile_size)
    if not isinstance(parsed_arguments.tile_overlap_size, int):
        parsed_arguments.tile_overlap_size = int(
            parsed_arguments.tile_overlap_size)

    tile_size = parsed_arguments.tile_size
    tile_overlap_size = parsed_arguments.tile_overlap_size

    data_format = parsed_arguments.data_format

    architecture = Architecture(parsed_architecture_json,
                                source_data_format='channels_last',
                                data_format=data_format)
    if architecture.data_format == 'channels_first':
        use_CPU_only = False
    else:
        use_CPU_only = True

    height = None
    width = None

    exr_files = OpenEXRDirectory._exr_files(parsed_arguments.input)
    features = {}
    required_features = architecture.auxiliary_features + architecture.feature_predictions
    for feature_prediction in required_features:
        exr_loaded = False

        if feature_prediction.load_data:
            for exr_file in exr_files:
                if feature_prediction.name in exr_file:
                    image = OpenEXRDirectory._load_exr(exr_file)

                    # HACK: Assume just one source input!
                    features[Naming.source_feature_name(
                        feature_prediction.name, index=0)] = image
                    exr_loaded = True

                    if height == None:
                        height = image.shape[0]
                        width = image.shape[1]
                    else:
                        assert height == image.shape[0]
                        assert width == image.shape[1]
                    break

        else:
            image = tf.ones(
                [height, width, feature_prediction.number_of_channels])
            if feature_prediction.feature_prediction_type != FeaturePredictionType.COLOR:
                # Direct and indirect need to be 0.5.
                image = tf.scalar_mul(0.5, image)
            features[Naming.source_feature_name(feature_prediction.name,
                                                index=0)] = image
            exr_loaded = True

        if not exr_loaded:
            # TODO: Improve (DeepBlender)
            raise Exception('Image for \'' + feature_prediction.name +
                            '\' could not be loaded or does not exist.')

    smaller_side_length = min(height, width)
    if smaller_side_length < 16:
        raise Exception(
            'The image needs to have at least a side length of 16 pixels.')

    if smaller_side_length < tile_size:
        ratio = tile_overlap_size / tile_size
        tile_size = smaller_side_length
        tile_overlap_size = int(tile_size * ratio)

    # Split the images into tiles.
    iteration_delta = tile_size - (2 * tile_overlap_size)

    width_count = width - (2 * tile_overlap_size) - (2 * iteration_delta)
    width_count = width_count / iteration_delta
    width_count = math.ceil(width_count) + 2

    height_count = height - (2 * tile_overlap_size) - (2 * iteration_delta)
    height_count = height_count / iteration_delta
    height_count = math.ceil(height_count) + 2

    tiled_features_grid = [[None for _ in range(width_count)]
                           for _ in range(height_count)]

    for height_index in range(height_count):
        if height_index == 0:
            lower_height = 0
            upper_height = tile_size
        elif height_index == height_count - 1:
            upper_height = height
            lower_height = upper_height - tile_size
        else:
            lower_height = height_index * iteration_delta
            upper_height = lower_height + tile_size

        for width_index in range(width_count):
            if width_index == 0:
                lower_width = 0
                upper_width = tile_size
            elif width_index == width_count - 1:
                upper_width = width
                lower_width = upper_width - tile_size
            else:
                lower_width = width_index * iteration_delta
                upper_width = lower_width + tile_size

            tiled_features = {}
            for feature_name in features:
                feature = features[feature_name]
                tiled_feature = feature[lower_height:upper_height,
                                        lower_width:upper_width]
                tiled_features[feature_name] = tiled_feature

            tiled_features_grid[height_index][width_index] = tiled_features

    # We don't need the features anymore.
    features = None

    if use_CPU_only:
        session_config = tf.ConfigProto(device_count={'GPU': 0})
    else:
        session_config = tf.ConfigProto()

    use_XLA = True
    if use_XLA:
        session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1

    run_config = tf.estimator.RunConfig(session_config=session_config)

    estimator = tf.estimator.Estimator(model_fn=model_fn,
                                       model_dir=architecture.model_directory,
                                       config=run_config,
                                       params={'architecture': architecture})

    tiled_features_list = []
    for height_index in range(height_count):
        for width_index in range(width_count):
            tiled_features = tiled_features_grid[height_index][width_index]
            tiled_features_list.append(tiled_features)

    predictions = estimator.predict(input_fn=lambda: input_fn_predict(
        tiled_features_list, tile_size, tile_size))

    for height_index in range(height_count):
        for width_index in range(width_count):
            tiled_features_grid[height_index][width_index] = next(predictions)

    predictions = {}
    for feature_prediction_tuple in architecture.feature_prediction_tuples:
        for feature_prediction in feature_prediction_tuple.feature_predictions:
            if feature_prediction.load_data:
                horizontal_feature_stripes = []
                for height_index in range(height_count):
                    horizontal_feature_elements = []
                    for width_index in range(width_count):
                        tiled_predictions = tiled_features_grid[height_index][
                            width_index]
                        prediction_name = Naming.feature_prediction_name(
                            feature_prediction.name)
                        prediction = tiled_predictions[prediction_name]

                        lower_height = 0
                        upper_height = tile_size
                        lower_width = 0
                        upper_width = tile_size

                        if width_index != 0 and width_index != width_count - 1:
                            lower_width = tile_overlap_size
                            upper_width = upper_width - tile_overlap_size
                        elif width_index == 0 and width_index == width_count - 1:
                            pass
                        elif width_index == 0:
                            upper_width = upper_width - tile_overlap_size
                        else:
                            assert width_index == width_count - 1
                            existing_width = tile_overlap_size + (
                                (width_count - 1) * (tile_size -
                                                     (2 * tile_overlap_size)))
                            remaining_width = width - existing_width
                            lower_width = upper_width - remaining_width

                        if height_index != 0 and height_index != height_count - 1:
                            lower_height = tile_overlap_size
                            upper_height = upper_height - tile_overlap_size
                        elif height_index == 0 and height_index == height_count - 1:
                            pass
                        elif height_index == 0:
                            upper_height = upper_height - tile_overlap_size
                        else:
                            assert height_index == height_count - 1
                            existing_height = tile_overlap_size + (
                                (height_count - 1) * (tile_size -
                                                      (2 * tile_overlap_size)))
                            remaining_height = height - existing_height
                            lower_height = upper_height - remaining_height

                        prediction = prediction[lower_height:upper_height,
                                                lower_width:upper_width]

                        horizontal_feature_elements.append(prediction)
                    if len(horizontal_feature_elements) > 1:
                        horizontal_feature_stripe = np.concatenate(
                            horizontal_feature_elements, 1)
                    else:
                        horizontal_feature_stripe = horizontal_feature_elements[
                            0]
                    horizontal_feature_stripes.append(
                        horizontal_feature_stripe)

                if len(horizontal_feature_stripes) > 1:
                    prediction = np.concatenate(horizontal_feature_stripes, 0)
                else:
                    prediction = horizontal_feature_stripes[0]
                prediction_name = Naming.feature_prediction_name(
                    feature_prediction.name)
                predictions[prediction_name] = prediction

    diffuse_direct = predictions[Naming.feature_prediction_name(
        RenderPasses.DIFFUSE_DIRECT)]
    diffuse_indirect = predictions[Naming.feature_prediction_name(
        RenderPasses.DIFFUSE_INDIRECT)]
    diffuse_color = predictions[Naming.feature_prediction_name(
        RenderPasses.DIFFUSE_COLOR)]

    glossy_direct = predictions[Naming.feature_prediction_name(
        RenderPasses.GLOSSY_DIRECT)]
    glossy_indirect = predictions[Naming.feature_prediction_name(
        RenderPasses.GLOSSY_INDIRECT)]
    glossy_color = predictions[Naming.feature_prediction_name(
        RenderPasses.GLOSSY_COLOR)]

    subsurface_direct = predictions[Naming.feature_prediction_name(
        RenderPasses.SUBSURFACE_DIRECT)]
    subsurface_indirect = predictions[Naming.feature_prediction_name(
        RenderPasses.SUBSURFACE_INDIRECT)]
    subsurface_color = predictions[Naming.feature_prediction_name(
        RenderPasses.SUBSURFACE_COLOR)]

    transmission_direct = predictions[Naming.feature_prediction_name(
        RenderPasses.TRANSMISSION_DIRECT)]
    transmission_indirect = predictions[Naming.feature_prediction_name(
        RenderPasses.TRANSMISSION_INDIRECT)]
    transmission_color = predictions[Naming.feature_prediction_name(
        RenderPasses.TRANSMISSION_COLOR)]

    volume_direct = predictions[Naming.feature_prediction_name(
        RenderPasses.VOLUME_DIRECT)]
    volume_indirect = predictions[Naming.feature_prediction_name(
        RenderPasses.VOLUME_INDIRECT)]

    environment = predictions[Naming.feature_prediction_name(
        RenderPasses.ENVIRONMENT)]
    emission = predictions[Naming.feature_prediction_name(
        RenderPasses.EMISSION)]

    # Combined features
    diffuse = np.multiply(diffuse_color,
                          np.add(diffuse_direct, diffuse_indirect))
    glossy = np.multiply(glossy_color, np.add(glossy_direct, glossy_indirect))
    subsurface = np.multiply(subsurface_color,
                             np.add(subsurface_direct, subsurface_indirect))
    transmission = np.multiply(
        transmission_color, np.add(transmission_direct, transmission_indirect))

    # Combined image
    image = np.add(diffuse, glossy)
    image = np.add(image, subsurface)
    image = np.add(image, transmission)
    image = np.add(image, volume_direct)
    image = np.add(image, volume_indirect)
    image = np.add(image, environment)
    image = np.add(image, emission)

    # TODO: Alpha currently ignored.

    # Store as npy to open in Blender.
    np.save(parsed_arguments.input + '/combined.npy', image)

    # HACK: Temporary output as png. (DeepBlender)
    image = 255. * image
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
    cv2.imwrite(parsed_arguments.input + '/combined.png', image,
                [int(cv2.IMWRITE_PNG_COMPRESSION), 9])
Exemplo n.º 39
0
                           dense_shape=[2, 4])
mat = tf.sparse.SparseTensor(
    indices=[[1, 0], [2, 0], [3, 0], [3, 1], [0, 2], [1, 3]],
    values=[0.85 * 0.33, 0.85 * 0.33, 0.85 * 0.33, 0.85, 0.85, 0.85],
    dense_shape=[4, 4])
neg_beta = 0.15 / 4
prev_rank = tf.get_variable("prev_rank",
                            dtype=float,
                            initializer=tf.constant([[1 / 4]
                                                     for _ in range(4)]))
curr_rank = tf.get_variable("curr_rank",
                            dtype=float,
                            initializer=tf.constant([[1 / 4]
                                                     for _ in range(4)]))
i = 0
with tf.Session() as sess:
    sess.run(init_op)
    while True:
        print(i)
        i += 1
        prev_rank = curr_rank
        curr_rank = tf.add(
            tf.sparse.sparse_dense_matmul(mat, prev_rank),
            tf.fill(dims=[4, 1],
                    value=tf.reduce_sum(tf.scalar_mul(neg_beta, prev_rank))))
        tf.global_variables_initializer().run()
        #print(sess.run(curr_rank))
        if sess.run(tf.reduce_sum(tf.abs(curr_rank - prev_rank))) < 0.00001:
            print("Here")
            print(sess.run(curr_rank))
            break
Exemplo n.º 40
0
def train():
        
    global_step = tf.Variable(0, name = 'global_step', trainable = False)

    train_dir = CURRENT_DIR + '/logs_without_condition/'
    data_dir = CURRENT_DIR + '/data/img_align_celeba_tfrecords/'

    images = inputs(data_dir, BATCH_SIZE)

    z = tf.placeholder(tf.float32, [None, Z_DIM], name='z')

    G = generator(z)
    _, D_logits  = discriminator(images)
    samples = sampler(z)
    _, D_logits_ = discriminator(G, reuse = True)
    
    d_loss_real = tf.reduce_mean(tf.scalar_mul(-1, D_logits))
    d_loss_fake = tf.reduce_mean(D_logits_)
    d_loss = d_loss_real + d_loss_fake
    g_loss = tf.reduce_mean(tf.scalar_mul(-1, D_logits_))
                                                                      
    z_sum = tf.summary.histogram('z', z)
    d_sum = tf.summary.histogram('d', D_logits)
    d__sum = tf.summary.histogram('d_', D_logits_)
    G_sum = tf.summary.image('G', G)

    d_loss_real_sum = tf.summary.scalar('d_loss_real', d_loss_real)
    d_loss_fake_sum = tf.summary.scalar('d_loss_fake', d_loss_fake)
    d_loss_sum = tf.summary.scalar('d_loss', d_loss)                                                
    g_loss_sum = tf.summary.scalar('g_loss', g_loss)
    
    g_sum = tf.summary.merge([z_sum, d__sum, G_sum, d_loss_fake_sum, g_loss_sum])
    d_sum = tf.summary.merge([z_sum, d_sum, d_loss_real_sum, d_loss_sum])

    t_vars = tf.trainable_variables()
    d_vars = [var for var in t_vars if 'd_' in var.name]
    g_vars = [var for var in t_vars if 'g_' in var.name]

    saver = tf.train.Saver()
    
    with tf.variable_scope("", reuse=tf.AUTO_REUSE):
        d_optim = tf.train.RMSPropOptimizer(LR).minimize(d_loss, var_list = d_vars, global_step = global_step)
        g_optim = tf.train.RMSPropOptimizer(LR).minimize(g_loss, var_list = g_vars, global_step = global_step)
    clip_d_op = [var.assign(tf.clip_by_value(var, CLIP[0], CLIP[1])) for var in d_vars]   
    
    #os.environ['CUDA_VISIBLE_DEVICES'] = str(0)
    #config = tf.ConfigProto()
    #config.gpu_options.per_process_gpu_memory_fraction = 0.2
    #sess = tf.InteractiveSession(config=config)
    sess = tf.Session()
     
    writer = tf.summary.FileWriter(train_dir, sess.graph)    
    
    sample_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM))
    
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess = sess, coord = coord)
    init = tf.initialize_all_variables()  
    sess.run(init)

    start = 0
    if LOAD_MODEL:        
        print(" [*] Reading checkpoints...")
        ckpt = tf.train.get_checkpoint_state(train_dir)        

        if ckpt and ckpt.model_checkpoint_path:
            ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
            saver.restore(sess, os.path.join(train_dir, ckpt_name))
            global_step = ckpt.model_checkpoint_path.split('/')[-1]\
                                                    .split('-')[-1]
            print('Loading success, global_step is %s' % global_step)
            
        start = int(global_step)
        
    for epoch in range(EPOCH):
        
        batch_idxs = 3072
        
        if epoch:
            start = 0
            
        for idx in range(start, batch_idxs):
            if idx<25 or idx % 500 == 0:
                critic_num = 25
            else:
                critic_num = CRITIC_NUM
            
            for _ in range(critic_num):
                batch_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM))
                
                _,summary_str = sess.run([d_optim,d_sum], feed_dict = {z:batch_z})
                
                sess.run(clip_d_op)
                
                writer.add_summary(summary_str,idx+1)
            
            batch_z = np.random.uniform(-1, 1, size = (BATCH_SIZE, Z_DIM))
            
            
            _, summary_str = sess.run([d_optim, d_sum], feed_dict = {z: batch_z})
            writer.add_summary(summary_str, idx+1)

            # Update G network
            _, summary_str = sess.run([g_optim, g_sum], feed_dict = {z: batch_z})
            writer.add_summary(summary_str, idx+1)

            # Run g_optim twice to make sure that d_loss does not go to zero
            _, summary_str = sess.run([g_optim, g_sum], feed_dict = {z: batch_z})
            writer.add_summary(summary_str, idx+1)
            
            errD_fake = 0#d_loss_fake.eval({z: batch_z})
            errD_real = 0#d_loss_real.eval()
            errG = 0#g_loss.eval({z: batch_z})
            if idx % 20 == 0:
                print("[%4d/%4d] d_loss: %.8f, g_loss: %.8f" \
                            % (idx, batch_idxs, errD_fake+errD_real, errG))
            
            if idx % 100 == 0:
                sample = sess.run(samples, feed_dict = {z: sample_z})
                samples_path = CURRENT_DIR + '\\out\\'
                save_images(sample, [8, 8], 
                            samples_path + \
                            'sample_%d_epoch_%d.png' % (epoch, idx))

                print('\n'*2)
                print('===========    %d_epoch_%d.png save down    ===========' 
                                                                %(epoch, idx))
                print('\n'*2)
                
            if (idx % 512 == 0) or (idx + 1 == batch_idxs):
                checkpoint_path = os.path.join(train_dir, 
                                               'my_dcgan_tfrecords.ckpt')
                saver.save(sess, checkpoint_path, global_step = idx+1)
                print('*********    model saved    *********')

        print('******* start with %d *******' % start)
    
    coord.request_stop()    
    coord.join(threads)
    sess.close()
Exemplo n.º 41
0
def main():
    """Create the model and start the training."""
    args = get_arguments()

    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)

    tf.set_random_seed(args.random_seed)

    # Create queue coordinator.
    coord = tf.train.Coordinator()

    # Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReader(args.data_dir, args.data_list, input_size,
                             args.random_scale, args.random_mirror, coord)
        image_batch, label_batch = reader.dequeue(args.batch_size)
        image_batch075 = tf.image.resize_images(
            image_batch, [int(h * 0.75), int(w * 0.75)])
        image_batch05 = tf.image.resize_images(
            image_batch, [int(h * 0.5), int(w * 0.5)])

    # Create network.
    with tf.variable_scope('', reuse=False):
        net = DeepLabResNetModel({'data': image_batch},
                                 is_training=args.is_training)
    with tf.variable_scope('', reuse=True):
        net075 = DeepLabResNetModel({'data': image_batch075},
                                    is_training=args.is_training)
    with tf.variable_scope('', reuse=True):
        net05 = DeepLabResNetModel({'data': image_batch05},
                                   is_training=args.is_training)
    # For a small batch size, it is better to keep
    # the statistics of the BN layers (running means and variances)
    # frozen, and to not update the values provided by the pre-trained model.
    # If is_training=True, the statistics will be updated during the training.
    # Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
    # if they are presented in var_list of the optimiser definition.

    # Predictions.
    raw_output100 = net.layers['fc1_voc12']
    raw_output075 = net075.layers['fc1_voc12']
    raw_output05 = net05.layers['fc1_voc12']
    raw_output = tf.reduce_max(tf.stack([
        raw_output100,
        tf.image.resize_images(raw_output075,
                               tf.shape(raw_output100)[1:3, ]),
        tf.image.resize_images(raw_output05,
                               tf.shape(raw_output100)[1:3, ])
    ]),
                               axis=0)
    # Which variables to load. Running means and variances are not trainable,
    # thus all_variables() should be restored.
    restore_var = tf.global_variables()
    all_trainable = [
        v for v in tf.trainable_variables()
        if 'beta' not in v.name and 'gamma' not in v.name
    ]
    fc_trainable = [v for v in all_trainable if 'fc' in v.name]
    conv_trainable = [v for v in all_trainable
                      if 'fc' not in v.name]  # lr * 1.0
    fc_w_trainable = [v for v in fc_trainable
                      if 'weights' in v.name]  # lr * 10.0
    fc_b_trainable = [v for v in fc_trainable
                      if 'biases' in v.name]  # lr * 20.0
    assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
    assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))

    # Predictions: ignoring all predictions with labels greater or equal than n_classes
    raw_prediction = tf.reshape(raw_output, [-1, n_classes])
    raw_prediction100 = tf.reshape(raw_output100, [-1, n_classes])
    raw_prediction075 = tf.reshape(raw_output075, [-1, n_classes])
    raw_prediction05 = tf.reshape(raw_output05, [-1, n_classes])

    label_proc = prepare_label(label_batch,
                               tf.pack(raw_output.get_shape()[1:3]),
                               one_hot=False)  # [batch_size, h, w]
    label_proc075 = prepare_label(label_batch,
                                  tf.pack(raw_output075.get_shape()[1:3]),
                                  one_hot=False)
    label_proc05 = prepare_label(label_batch,
                                 tf.pack(raw_output05.get_shape()[1:3]),
                                 one_hot=False)

    raw_gt = tf.reshape(label_proc, [
        -1,
    ])
    raw_gt075 = tf.reshape(label_proc075, [
        -1,
    ])
    raw_gt05 = tf.reshape(label_proc05, [
        -1,
    ])

    indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, n_classes - 1)), 1)
    indices075 = tf.squeeze(tf.where(tf.less_equal(raw_gt075, n_classes - 1)),
                            1)
    indices05 = tf.squeeze(tf.where(tf.less_equal(raw_gt05, n_classes - 1)), 1)

    gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
    gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32)
    gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32)

    prediction = tf.gather(raw_prediction, indices)
    prediction100 = tf.gather(raw_prediction100, indices)
    prediction075 = tf.gather(raw_prediction075, indices075)
    prediction05 = tf.gather(raw_prediction05, indices05)

    # Pixel-wise softmax loss.
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
                                                          labels=gt)
    loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=prediction100, labels=gt)
    loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=prediction075, labels=gt075)
    loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=prediction05, labels=gt05)
    l2_losses = [
        args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables()
        if 'weights' in v.name
    ]
    reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean(
        loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(loss05) + tf.add_n(
            l2_losses)

    # Processed predictions: for visualisation.
    raw_output_up = tf.image.resize_bilinear(raw_output,
                                             tf.shape(image_batch)[1:3, ])
    raw_output_up = tf.argmax(raw_output_up, dimension=3)
    pred = tf.expand_dims(raw_output_up, dim=3)

    # Image summary.
    images_summary = tf.py_func(inv_preprocess,
                                [image_batch, args.save_num_images], tf.uint8)
    labels_summary = tf.py_func(decode_labels,
                                [label_batch, args.save_num_images], tf.uint8)
    preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images],
                               tf.uint8)

    total_summary = tf.summary.image(
        'images',
        tf.concat(2, [images_summary, labels_summary, preds_summary]),
        max_outputs=args.save_num_images)  # Concatenate row-wise.
    summary_writer = tf.summary.FileWriter(args.snapshot_dir,
                                           graph=tf.get_default_graph())

    # Define loss and optimisation parameters.
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(
        base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))

    opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
    opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
    opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)

    # Define a variable to accumulate gradients.
    accum_grads = [
        tf.Variable(tf.zeros_like(v.initialized_value()), trainable=False)
        for v in conv_trainable + fc_w_trainable + fc_b_trainable
    ]

    # Define an operation to clear the accumulated gradients for next batch.
    zero_op = [v.assign(tf.zeros_like(v)) for v in accum_grads]

    # Compute gradients.
    grads = tf.gradients(reduced_loss,
                         conv_trainable + fc_w_trainable + fc_b_trainable)

    # Accumulate and normalise the gradients.
    accum_grads_op = [
        accum_grads[i].assign_add(grad / args.grad_update_every)
        for i, grad in enumerate(grads)
    ]

    grads_conv = accum_grads[:len(conv_trainable)]
    grads_fc_w = accum_grads[len(conv_trainable):(len(conv_trainable) +
                                                  len(fc_w_trainable))]
    grads_fc_b = accum_grads[(len(conv_trainable) + len(fc_w_trainable)):]

    # Apply the gradients.
    train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
    train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
    train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))

    train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)

    # Set up tf session and initialize variables.
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()

    sess.run(init)

    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=restore_var, max_to_keep=10)

    # Load variables if the checkpoint is provided.
    if args.restore_from is not None:
        loader = tf.train.Saver(var_list=restore_var)
        load(loader, sess, args.restore_from)

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        feed_dict = {step_ph: step}
        loss_value = 0

        # Clear the accumulated gradients.
        sess.run(zero_op, feed_dict=feed_dict)

        # Accumulate gradients.
        for i in range(args.grad_update_every):
            _, l_val = sess.run([accum_grads_op, reduced_loss],
                                feed_dict=feed_dict)
            loss_value += l_val

        # Normalise the loss.
        loss_value /= args.grad_update_every

        # Apply gradients.
        if step % args.save_pred_every == 0:
            images, labels, summary, _ = sess.run(
                [image_batch, label_batch, total_summary, train_op],
                feed_dict=feed_dict)
            summary_writer.add_summary(summary, step)
            save(saver, sess, args.snapshot_dir, step)
        else:
            sess.run(train_op, feed_dict=feed_dict)

        duration = time.time() - start_time
        print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(
            step, loss_value, duration))
    coord.request_stop()
    coord.join(threads)
def train(target, dataset, cluster_spec):
  """Train Inception on a dataset for a number of steps."""
  # Number of workers and parameter servers are inferred from the workers and ps
  # hosts string.
  num_workers = len(cluster_spec.as_dict()['worker'])
  num_parameter_servers = len(cluster_spec.as_dict()['ps'])
  # If no value is given, num_replicas_to_aggregate defaults to be the number of
  # workers.
  if FLAGS.num_replicas_to_aggregate == -1:
    num_replicas_to_aggregate = num_workers
  else:
    num_replicas_to_aggregate = FLAGS.num_replicas_to_aggregate

  # Both should be greater than 0 in a distributed training.
  assert num_workers > 0 and num_parameter_servers > 0, (' num_workers and '
                                                         'num_parameter_servers'
                                                         ' must be > 0.')

  # Choose worker 0 as the chief. Note that any worker could be the chief
  # but there should be only one chief.
  is_chief = (FLAGS.task_id == 0)

  #batchSizeManager = BatchSizeManager(32, 4)

  # Ops are assigned to worker by default.
  tf.logging.info('cccc-num_parameter_servers:'+str(num_parameter_servers))
  partitioner = tf.fixed_size_partitioner(num_parameter_servers, 0)  

  device_setter = tf.train.replica_device_setter(ps_tasks=num_parameter_servers)
  slim = tf.contrib.slim
  with tf.device('/job:worker/task:%d' % FLAGS.task_id):
   with tf.variable_scope('root', partitioner=partitioner):
    # Variables and its related init/assign ops are assigned to ps.
#    with slim.arg_scope(
#        [slim.variables.variable, slim.variables.global_step],
#        device=slim.variables.VariableDeviceChooser(num_parameter_servers)):
    with tf.device(device_setter):
#	partitioner=partitioner):
      # Create a variable to count the number of train() calls. This equals the
      # number of updates applied to the variables.
#      global_step = slim.variables.global_step()
      global_step = tf.Variable(0, trainable=False)

      # Calculate the learning rate schedule.

      batch_size = tf.placeholder(dtype=tf.int32, shape=(), name='batch_size')
      num_batches_per_epoch = (dataset.num_examples_per_epoch() /
                               FLAGS.batch_size)
      # Decay steps need to be divided by the number of replicas to aggregate.
      decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay /
                        num_replicas_to_aggregate)

      # Decay the learning rate exponentially based on the number of steps.
      lr = tf.train.exponential_decay(FLAGS.initial_learning_rate,
                                      global_step,
                                      decay_steps,
                                      FLAGS.learning_rate_decay_factor,
                                      staircase=True)
      # Add a summary to track the learning rate.
#      tf.summary.scalar('learning_rate', lr)

      # Create an optimizer that performs gradient descent.
      opt = tf.train.RMSPropOptimizer(lr,
                                      RMSPROP_DECAY,
                                      momentum=RMSPROP_MOMENTUM,
                                      epsilon=RMSPROP_EPSILON)

      images, labels = image_processing.distorted_inputs(
          dataset,
          batch_size,
          num_preprocess_threads=FLAGS.num_preprocess_threads)
      print(images.get_shape())
      print(labels.get_shape())

      # Number of classes in the Dataset label set plus 1.
      # Label 0 is reserved for an (unused) background class.
#      num_classes = dataset.num_classes() + 1
      num_classes = dataset.num_classes()
      print(num_classes)
#      logits = inception.inference(images, num_classes, for_training=True)
      network_fn = nets_factory.get_network_fn('inception_v3',num_classes=num_classes) 
      (logits,_) = network_fn(images)
      print(logits.get_shape())
      # Add classification loss.
#      inception.loss(logits, labels, batch_size)

      # Gather all of the losses including regularization losses.
      labels = tf.one_hot(labels, 1000, 1, 0)
      cross_entropy = tf.losses.softmax_cross_entropy(
          logits=logits, 
          onehot_labels=labels)
#      losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
#      losses += tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
      losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
      total_loss = cross_entropy + _WEIGHT_DECAY * tf.add_n(
          [tf.nn.l2_loss(v) for v in tf.trainable_variables()])

#      total_loss = tf.add_n(losses, name='total_loss')

      if is_chief:
        # Compute the moving average of all individual losses and the
        # total loss.
        loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
        loss_averages_op = loss_averages.apply(losses + [total_loss])

        # Attach a scalar summmary to all individual losses and the total loss;
        # do the same for the averaged version of the losses.
#        for l in losses + [total_loss]:
#          loss_name = l.op.name
          # Name each loss as '(raw)' and name the moving average version of the
          # loss as the original loss name.
#          tf.summary.scalar(loss_name + ' (raw)', l)
#          tf.summary.scalar(loss_name, loss_averages.average(l))

        # Add dependency to compute loss_averages.
        with tf.control_dependencies([loss_averages_op]):
          total_loss = tf.identity(total_loss)

      # Track the moving averages of all trainable variables.
      # Note that we maintain a 'double-average' of the BatchNormalization
      # global statistics.
      # This is not needed when the number of replicas are small but important
      # for synchronous distributed training with tens of workers/replicas.
      exp_moving_averager = tf.train.ExponentialMovingAverage(
          MOVING_AVERAGE_DECAY, global_step)

      variables_to_average = (
          tf.trainable_variables() + tf.moving_average_variables())

      # Add histograms for model variables.
#      for var in variables_to_average:
#        tf.summary.histogram(var.op.name, var)

      # Create synchronous replica optimizer.
      opt = tf.train.SyncReplicasOptimizer(
          opt,
          replicas_to_aggregate=num_replicas_to_aggregate,
          total_num_replicas=num_workers,
          variable_averages=exp_moving_averager,
          variables_to_average=variables_to_average)

#      batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION)
#      assert batchnorm_updates, 'Batchnorm updates are missing'
#      batchnorm_updates_op = tf.group(*batchnorm_updates)
#      # Add dependency to compute batchnorm_updates.
#      with tf.control_dependencies([batchnorm_updates_op]):
#        total_loss = tf.identity(total_loss)

      # Compute gradients with respect to the loss.
      # grads = opt.compute_gradients(total_loss)
      grads0 = opt.compute_gradients(total_loss) 
      grads = [(tf.scalar_mul(tf.cast(batch_size/FLAGS.batch_size, tf.float32), grad), var) for grad, var in grads0]

      # Add histograms for gradients.
#      for grad, var in grads:
#        if grad is not None:
#          tf.summary.histogram(var.op.name + '/gradients', grad)

      apply_gradients_op = opt.apply_gradients(grads, global_step=global_step)

      with tf.control_dependencies([apply_gradients_op]):
        train_op = tf.identity(total_loss, name='train_op')

      # Get chief queue_runners and init_tokens, which is used to synchronize
      # replicas. More details can be found in SyncReplicasOptimizer.
      chief_queue_runners = [opt.get_chief_queue_runner()]
      init_tokens_op = opt.get_init_tokens_op()

      # Create a saver.
      saver = tf.train.Saver()

      # Build the summary operation based on the TF collection of Summaries.
#      summary_op = tf.summary.merge_all()

      # Build an initialization operation to run below.
      init_op = tf.global_variables_initializer()

      # We run the summaries in the same thread as the training operations by
      # passing in None for summary_op to avoid a summary_thread being started.
      # Running summaries and training operations in parallel could run out of
      # GPU memory.
      sv = tf.train.Supervisor(is_chief=is_chief,
                               logdir=FLAGS.train_dir,
                               init_op=init_op,
                               summary_op=None,
                               global_step=global_step,
                               recovery_wait_secs=1,
                               saver=None,
                               save_model_secs=FLAGS.save_interval_secs)

      tf.logging.info('%s Supervisor' % datetime.now())

      sess_config = tf.ConfigProto(
          allow_soft_placement=True,
          log_device_placement=FLAGS.log_device_placement)

      # Get a session.
      sess = sv.prepare_or_wait_for_session(target, config=sess_config)

      # Start the queue runners.
      queue_runners = tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS)
      sv.start_queue_runners(sess, queue_runners)
      tf.logging.info('Started %d queues for processing input data.',
                      len(queue_runners))

      if is_chief:
        sv.start_queue_runners(sess, chief_queue_runners)
        sess.run(init_tokens_op)

      # Train, checking for Nans. Concurrently run the summary operation at a
      # specified interval. Note that the summary_op and train_op never run
      # simultaneously in order to prevent running out of GPU memory.
#      next_summary_time = time.time() + FLAGS.save_summaries_secs
      step = 0
      time0 = time.time()
      batch_size_num = 1
      while not sv.should_stop():
        try:
          start_time = time.time()

	  batch_size_num = 32
#	   batch_size_num = int((int(step)/3*10)) % 100000 + 1
#          if step < 5:
#            batch_size_num = 32 
#          batch_size_num = (batch_size_num ) % 64 + 1
#          else:
#            batch_size_num = 80

          run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
          run_metadata = tf.RunMetadata()

          my_images, loss_value, step = sess.run([images, train_op, global_step], feed_dict={batch_size: batch_size_num}, options=run_options, run_metadata=run_metadata)
	  b = time.time()
#          assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
          if step > FLAGS.max_steps:
            break
          duration = time.time() - start_time
	  thread = threading2.Thread(target=get_computation_time, name="get_computation_time",args=(run_metadata.step_stats,step,))
	  thread.start()
#          tl = timeline.Timeline(run_metadata.step_stats)
#          last_batch_time = tl.get_local_step_duration('sync_token_q_Dequeue')
          c0 = time.time()
#          batch_size_num = batchSizeManager.dictate_new_batch_size(FLAGS.task_id, last_batch_time)
#          batch_size_num = rpcClient.update_batch_size(FLAGS.task_id, last_batch_time, available_cpu, available_memory, step, batch_size_num) 
#          ctf = tl.generate_chrome_trace_format()
#          with open("timeline.json", 'a') as f:
#            f.write(ctf)

          if step % 1 == 0:
            examples_per_sec = FLAGS.batch_size / float(duration)
            c = time.time()
            tf.logging.info("time statistics" + " - train_time: " + str(b-start_time) + " - get_batch_time: " + str(c0-b) + " - get_bs_time:  " + str(c-c0) + " - accum_time: " + str(c-time0) + " - batch_size: " + str(batch_size_num))
            format_str = ('Worker %d: %s: step %d, loss = %.2f'
                          '(%.1f examples/sec; %.3f  sec/batch)')
            tf.logging.info(format_str %
                            (FLAGS.task_id, datetime.now(), step, loss_value,
                             examples_per_sec, duration))

          # Determine if the summary_op should be run on the chief worker.
#          if is_chief and next_summary_time < time.time():
#            tf.logging.info('Running Summary operation on the chief.')
#            summary_str = sess.run(summary_op)
#            sv.summary_computed(sess, summary_str)
#            tf.logging.info('Finished running Summary operation.')

            # Determine the next time for running the summary.
#            next_summary_time += FLAGS.save_summaries_secs
        except:
          if is_chief:
            tf.logging.info('Chief got exception while running!')
          raise

      # Stop the supervisor.  This also waits for service threads to finish.
      sv.stop()
Exemplo n.º 43
0
  def apply_gradients(self, grads_and_vars, worker_id, global_step=None, name=None, collect_cdfs=False,
    #  batch_idx_list=None, worker_kill_list=None, num_workers=None, num_batches_per_epoch=None):
    matrix_to_solve=None, num_batches_per_epoch=None):
    """Apply gradients to variables.
    This contains most of the synchronization implementation and also wraps the
    apply_gradients() from the real optimizer.
    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        compute_gradients().
      global_step: Optional Variable to increment by one after the
        variables have been updated.
      name: Optional name for the returned operation.  Default to the
        name passed to the Optimizer constructor.
    Returns:
      train_op: The op to dequeue a token so the replicas can exit this batch
      and start the next one. This is executed by each replica.
    Raises:
      ValueError: If the grads_and_vars is empty.
      ValueError: If global step is not provided, the staleness cannot be
        checked.
    """
    if not grads_and_vars:
      raise ValueError("Must supply at least one variable")

    if global_step is None:
      raise ValueError("Global step is required to check staleness")

    self._global_step = global_step
    train_ops = []
    aggregated_grad = []
    var_list = []

    self._local_step = variables.Variable(
        initial_value=0,
        trainable=False,
        collections=[ops.GraphKeys.LOCAL_VARIABLES],
        dtype=global_step.dtype.base_dtype,
        name="sync_rep_local_step")
    self.local_step_init_op = state_ops.assign(self._local_step, global_step._ref())
    chief_init_ops = [self.local_step_init_op]
    self.ready_for_local_init_op = variables.report_uninitialized_variables(
      variables.all_variables())

    # The wait op waits for the current worker to dequeue a token from its respective token queue
    self._wait_op = self._sync_token_queues[worker_id].dequeue()

    # Replicas have to wait until they can get a token from the token queue
    # BEFORE begining to compute gradients.
    with ops.device(global_step.device):
      queue_size = self._sync_token_queues[worker_id].size()
      update_local_step_op = state_ops.assign(self._local_step, global_step._ref())

    # Gradient accum creation
    with ops.name_scope(None, self._name):
      for grad, var in grads_and_vars:
        var_list.append(var)
        tf.logging.info("Grad " + str(grad) + " assigned to " + str(var.device))
        with ops.device(var.device):
          if grad is None:
            continue
          elif isinstance(grad, ops.Tensor):
            grad_accum = data_flow_ops.ConditionalAccumulator(
              grad.dtype,
              shape=var.get_shape(),
              shared_name=var.name + "/grad_accum")
          else:
            if not isinstance(grad, ops.IndexedSlices):
              raise ValueError("Unknown grad type!")
            grad_accum = data_flow_ops.SparseConditionalAccumulator(
              grad.dtype, shape=(), shared_name=var.name + "/grad_accum")

          self._accumulator_list.append((grad_accum, var))

      """# Phase 1 gradient computation
      with ops.control_dependencies([update_local_step_op]):
        for index, (grad, var) in enumerate(grads_and_vars):
          with ops.device(var.device):
            if grad is None:
              continue
            elif isinstance(grad, ops.Tensor):
              grad_accum = self._accumulator_list[index][0]
              train_ops.append(grad_accum.apply_grad(grad,
                                                     local_step=self._local_step._ref()))
            else:
              if not isinstance(grad, ops.IndexedSlices):
                raise ValueError("Unknown grad type!")
              grad_accum = self._accumulator_list[index][0]
              train_ops.append(grad_accum.apply_indexed_slices_grad(
                grad, local_step=self._local_step._ref()))"""

      # Phase 1 gradient computation
      with ops.control_dependencies([update_local_step_op]):
        for index, (grad, var) in enumerate(grads_and_vars):
          print_start_op = logging_ops.Print(global_step, [global_step], message="Starting to apply grads for variable %d" % index)
          train_ops.append(print_start_op)
          with ops.device(var.device):
            ps_step_printer0 = logging_ops.Print(global_step, [global_step], message="global step printer0 on ps")
            train_ops.append(ps_step_printer0)
            '''Implement LS computation and solution here'''            
            #b = np.ones(int(num_batches_per_epoch))
            b = tf.ones([int(num_batches_per_epoch),1], tf.float32)         
            A = matrix_to_solve
#            A_for_calc = np.transpose(A)
            LS_solution = linalg_ops.matrix_solve_ls(A, b, fast=False)
            LS_calc = tf.reshape(LS_solution, [-1])
            weight = tf.slice(LS_calc, [worker_id], [1])
#            print_ls_op = logging_ops.Print(LS_calc, [LS_calc], message="Solution for LS!")
#            train_ops.append(print_ls_op)
            weighted_grad = tf.scalar_mul(weight[0], grad)
            '''Kill some workers'''
            if grad is None:
              continue

            elif isinstance(grad, ops.Tensor):
              grad_accum = self._accumulator_list[index][0]

              num_accum = grad_accum.num_accumulated()
              tf.logging.info("Grad Accumed %s, Worker ID: %s" % (str(num_accum), str(worker_id)))

              with ops.control_dependencies([print_start_op]):
                with tf.device("job:worker/task:%d" % worker_id):
                  apply_grad_op = grad_accum.apply_grad(grad,
#                  apply_grad_op = grad_accum.apply_grad(weighted_grad,
                                                        local_step=self._local_step._ref())
                  with ops.control_dependencies([apply_grad_op]):
                    finished_print_op = logging_ops.Print(global_step, [global_step], message="Done applying grads for variable %d" % index)
                    train_ops.append(finished_print_op)

            else:
              if not isinstance(grad, ops.IndexedSlices):
                raise ValueError("Unknown grad type!")
              grad_accum = self._accumulator_list[index][0]

              with ops.control_dependencies([print_start_op]):
                with tf.device("job:worker/task:%d" % worker_id):
                  apply_grad_op = grad_accum.apply_indexed_slices_grad(
                    grad, local_step=self._local_step._ref())
#                    weighted_grad, local_step=self._local_step._ref())
                  with ops.control_dependencies([apply_grad_op]):
                    finished_print_op = logging_ops.Print(global_step, [global_step], message="Done applying grads for variable %d" % index)
                    train_ops.append(finished_print_op)

      # Phase 2 gradient applying
      for index, (grad, var) in enumerate(grads_and_vars):
        with ops.device(var.device):
          work_idx_print1 = logging_ops.Print(worker_id, [worker_id], message="worker id for aggregate grad")
          ps_step_printer1 = logging_ops.Print(global_step, [global_step], message="global step printer1 on ps")
          num_replica_aggragate = logging_ops.Print(self._replicas_to_aggregate, [self._replicas_to_aggregate], message="num replica aggregate")
          train_ops.append(work_idx_print1)
          train_ops.append(ps_step_printer1)
          train_ops.append(num_replica_aggragate)
          grad_accum = self._accumulator_list[index][0]
       
          if grad is None:
            aggregated_grad.append(None)
          elif isinstance(grad, ops.Tensor):
            if collect_cdfs:
#              aggregated_grad.append(grad_accum.take_grad(self._total_num_replicas))
              aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate))
            else:
              aggregated_grad.append(grad_accum.take_grad(1))
          else:
            if collect_cdfs:
#              aggregated_grad.append(grad_accum.take_grad(self._total_num_replicas))
              aggregated_grad.append(grad_accum.take_grad(self._replicas_to_aggregate))
            else:
              aggregated_grad.append(grad_accum.take_indexed_slices_grad(1))

      aggregated_grads_and_vars = zip(aggregated_grad, var_list)

      # Some debug operations
      self.print_sizes = logging_ops.Print(global_step, [self._sync_token_queues[i].size() for i in range(self._total_num_replicas)], message="queue sizes")
      self.print_accum_sizes = logging_ops.Print(self._local_step,
                                                 [x[0].num_accumulated() for x in self._accumulator_list] + [worker_id],
                                                 message="Accum sizes")
      self.print_local_step = logging_ops.Print(self._local_step, [self._local_step._ref(), global_step._ref()], message="local vs global step")

      # sync_op will be assigned to the same device as the global step.
      with ops.device(global_step.device), ops.name_scope(""):
        with ops.control_dependencies([self.print_accum_sizes]):
          update_op = self._opt.apply_gradients(aggregated_grads_and_vars, global_step)
          self._update_op = update_op
          with ops.control_dependencies([update_op]):
            sync_op = []
            for cur_worker_id in range(self._total_num_replicas):
              sync_op.append(self._sync_token_queues[cur_worker_id].enqueue(global_step))
            sync_op = control_flow_ops.group(*(sync_op))

        # dummy_queue is passed to the queue runner. Don't use the real queues
        # because the queue runner doesn't automatically reopen it once it
        # closed queues in PS devices.
        dummy_queue = (
            data_flow_ops.FIFOQueue(1,
                                    types_pb2.DT_INT32,
                                    shapes=(),
                                    shared_name="dummy_queue"))

        self._chief_queue_runner = queue_runner.QueueRunner(dummy_queue,
                                                            [sync_op])

      with ops.device(global_step.device), ops.name_scope(""):
        with ops.control_dependencies(train_ops):
          # Worker finished applying gradients. Add token to phase1_finished_queue
          train_op = logging_ops.Print(self._local_step._ref(),
                                       [x[0].num_accumulated() for x in self._accumulator_list] + [worker_id],
                                       message="Finished worker updates",
                                       name="FinishedWorkerUpdatesPrint")

      for accum, var in self._accumulator_list:
        with ops.device(var.device):
          chief_init_ops.append(
              accum.set_global_step(
                  global_step, name="SetGlobalStep"))
      self.chief_init_op = control_flow_ops.group(*(chief_init_ops))
      self._gradients_applied = True

      return train_op
Exemplo n.º 44
0
def main():
    """Create the model and start the training."""
    args = get_arguments()
    args.snapshot_dir=args.snapshot_dir.replace('DeepDICD/','DeepDICD/'+args.model_name+'-'+args.domain+'-')
    print(toMagenta(args.snapshot_dir))
    ss=args.domain.split('-')
    if ss[0]=='D':
        args.list1=args.list1.replace('amazon.txt','dslr.txt')
    elif ss[0]=='W':
        args.list1=args.list1.replace('amazon.txt','webcam.txt')

    if ss[1]=='A':
        args.list2=args.list2.replace('dslr.txt','amazon.txt')
    elif ss[1]=='W':
        args.list2=args.list2.replace('dslr.txt','webcam.txt')

    print(toMagenta(args.list1))
    print(toMagenta(args.list2))

    start_steps=args.start_steps
    
    h=args.h
    w=args.w

    # construct data generator
    file1 = open(args.list1) 
    num1 = len(file1.readlines()) 
    file2 = open(args.list2) 
    num2 = len(file2.readlines())
    file1.close()
    file2.close() 

    steps_per_epoch=int((num1/(args.batch_size)))
    num_steps=int(steps_per_epoch*args.num_epochs)
    val_num_steps=int(num2/args.batch_size)

    print(toCyan('src domain: {:d}, tar domain {:d}'.format(num1,num2)))
    print(toCyan('steps_per_epoch x num_epochs:{:d} x {:d}'.format(steps_per_epoch,args.num_epochs)))

    # Chong
    # split_batch_size=int(args.batch_size/hvd.size()) 
    myDataloader=Dataloader(args.img_dir,args.list1,args.list2,args.batch_size,args.h,args.w,args.num_threads)

    src_img=myDataloader.simg_batch
    src_label=myDataloader.slabel_batch
    tar_img=myDataloader.timg_batch
    tar_label=myDataloader.tlabel_batch

    coord = tf.train.Coordinator()

    # Learning Startegy Settings
    baseLR1 = tf.constant(args.lr1)
    baseLR2 = tf.constant(args.lr2)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    if args.learning_strategy==0:
        lr1=baseLR1/tf.pow(1+0.001*step_ph/steps_per_epoch,0.75)
        lr2=baseLR2/tf.pow(1+0.001*step_ph/steps_per_epoch,0.75)
    elif args.learning_strategy==1:
        lr1=baseLR1/tf.pow(1+0.0005*step_ph/steps_per_epoch,0.75)
        lr2=baseLR2/tf.pow(1+0.0005*step_ph/steps_per_epoch,0.75)
    if args.learning_strategy==2:
        lr1=baseLR1/tf.pow(1+0.005*step_ph/steps_per_epoch,0.75)
        lr2=baseLR2/tf.pow(1+0.005*step_ph/steps_per_epoch,0.75)
    if args.learning_strategy==3:
        lr1=baseLR1/tf.pow(1+0.001*step_ph,0.75)
        lr2=baseLR2/tf.pow(1+0.001*step_ph,0.75)
    

    
    # lr1 = tf.scalar_mul(baseLR1, tf.pow((1 - step_ph / num_steps), args.power))
    # lr2 = tf.scalar_mul(baseLR2, tf.pow((1 - step_ph / num_steps), args.power))
    
    # lr1=baseLR1/tf.pow(1+0.001*step_ph,0.75)
    # lr2=baseLR2/tf.pow(1+0.001*step_ph,0.75)
    # Polynomial_decay 
    

    # lr1=baseLR1
    # lr2=baseLR2
    # decay_steps=steps_per_epoch*10
    # lr1=tf.train.exponential_decay(baseLR1,step_ph,decay_steps,0.1,staircase=True)
    # lr2=tf.train.exponential_decay(baseLR2,step_ph,decay_steps,0.1,staircase=True)
    keep_prob=tf.placeholder(dtype=tf.float32,shape=())
    p=(1- tf.scalar_mul(1., tf.pow((1 - step_ph / num_steps), args.power)))
    alpha1 = adaptation_factor(step_ph*1.0/10000)
    # alpha1 = tf.constant(1.,tf.float32)
    alpha2 = 0.01*p
    # alpha2 = 0
   

    model = DeepDICDModel(args,keep_prob,src_img,src_label,tar_img,tar_label)
    model.build_losses() 
    model.build_outputs()

    all_trainable_var=[v for v in tf.trainable_variables()]
    fine_tune_var = [v for v in all_trainable_var if 'fc8' not in v.name and 'fc9' not in v.name and 'domain' not in v.name]
    fine_tune_var_weights=[v for v in fine_tune_var if 'weights' in v.name]
    fine_tune_var_bias=[v for v in fine_tune_var if 'bias' in v.name]
    retrain_var = [v for v in all_trainable_var if 'fc8' in v.name or 'fc9' in v.name]
    retrain_var_weights=[v for v in retrain_var if 'weights' in v.name]
    retrain_var_bias=[v for v in retrain_var if 'bias' in v.name]
    domain_var=[v for v in all_trainable_var if 'domain' in v.name]
    domain_var_weights=[v for v in domain_var if 'weights' in v.name]
    domain_var_bias=[v for v in domain_var if 'bias' in v.name]
    print(toGreen(domain_var_bias))
    model.build_regular_losses(domain_var_weights,fine_tune_var_weights+fine_tune_var_weights)
    summary_=model.build_summary()

    # ------------- Loss Function ------------------
    # F_loss=model.classify_loss_src+model.dregular_loss+model.G_loss \
    # + model.class_wise_adaptation_loss \
    # + alpha2*(model.sintra_loss+model.tintra_loss-0.1*model.sinter_loss-0.1*model.tinter_loss)
    # F_loss=model.classify_loss_src+model.dregular_loss+alpha1*model.G_loss \
    # + alpha1*model.class_wise_adaptation_loss \
    # + alpha2*(model.sintra_loss+model.tintra_loss-0.1*model.sinter_loss-0.1*model.tinter_loss)
    F_loss=model.classify_loss_src+model.gregular_loss+alpha1*model.G_loss \
    + alpha1*model.class_wise_adaptation_loss \
    + alpha2*(model.sintra_loss+model.tintra_loss)


    # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
    if args.no_update_mean_var == True:
        update_ops = None
    else:
        print(toMagenta('updating mean and var in batchnorm'))
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    with tf.control_dependencies(update_ops):

        opt_1_1 = tf.train.MomentumOptimizer(lr1, args.momentum)
        grads_1_1 = tf.gradients(F_loss, fine_tune_var_weights)
        train_op_1_1 = opt_1_1.apply_gradients(zip(grads_1_1, fine_tune_var_weights))
        
        
        opt_1_2 = tf.train.MomentumOptimizer(2*lr1, args.momentum)
        grads_1_2 = tf.gradients(F_loss, fine_tune_var_bias)
        train_op_1_2 = opt_1_2.apply_gradients(zip(grads_1_2, fine_tune_var_bias))
        
       
        opt_2_1 = tf.train.MomentumOptimizer(lr2, args.momentum)
        grads_2_1 = tf.gradients(F_loss, retrain_var_weights)
        train_op_2_1 = opt_2_1.apply_gradients(zip(grads_2_1, retrain_var_weights))
        
       
        opt_2_2 = tf.train.MomentumOptimizer(2*lr2, args.momentum)
        grads_2_2 = tf.gradients(F_loss, retrain_var_bias)
        train_op_2_2 = opt_2_2.apply_gradients(zip(grads_2_2, retrain_var_bias))
        

        opt_3_1 = tf.train.MomentumOptimizer(lr2,args.momentum)
        grads_3_1 = tf.gradients(model.dregular_loss+model.D_loss, domain_var_weights)
        train_op_3_1 = opt_3_1.apply_gradients(zip(grads_3_1, domain_var_weights))
        

        opt_3_2 = tf.train.MomentumOptimizer(2*lr2,args.momentum)
        grads_3_2 = tf.gradients(model.dregular_loss+model.D_loss, domain_var_bias)
        train_op_3_2 = opt_3_2.apply_gradients(zip(grads_3_2, domain_var_bias))
        

        train_op = tf.group(train_op_1_1,train_op_1_2,train_op_2_1,train_op_2_2,train_op_3_1,train_op_3_2)
    # Set up tf session and initialize variables. 
    # 
    config = tf.ConfigProto()#Chong
    config.gpu_options.allow_growth = True
    config.gpu_options.per_process_gpu_memory_fraction = 0.4
    sess = tf.Session(config=config)
    init_local=tf.local_variables_initializer()
    init = tf.global_variables_initializer()

   
    # construct summary
    summary_.append(tf.summary.scalar(
        'train/lr1', lr1))
    summary_.append(tf.summary.scalar(
        'train/lr2', lr2))
    summary_.append(tf.summary.scalar(
        'train/alpha1', alpha1))
    summary_.append(tf.summary.scalar(
        'train/alpha2', alpha2))

    summary_merged=tf.summary.merge(summary_)
    FinalSummary = tf.summary.FileWriter(args.snapshot_dir,sess.graph)

    # init
    sess.run([init_local,init])
  
    # Saver for storing checkpoints of the model.
    var=tf.global_variables()
    skip_var=['fc8','fc9']
    saver = tf.train.Saver(var_list=var, max_to_keep=5)

    ckpt = tf.train.get_checkpoint_state(args.resume_from)
    if ckpt and ckpt.model_checkpoint_path and args.resume:
        loader = tf.train.Saver(var_list=var)
        load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
        load(loader, sess, ckpt.model_checkpoint_path)
    elif not args.not_load_pretrained:
        print(toRed('Restore from pre-trained model...' + args.restore_from))
        model.load_initial_weights(sess, args.restore_from, skip_var) #Chong:0531

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    acc2_history=0
    for step in range(start_steps,num_steps):
        start_time = time.time()
        feed_dict = {step_ph: step, keep_prob: 0.5}
        summary, total_loss, _= sess.run([summary_merged,F_loss,train_op], feed_dict=feed_dict)
        
        FinalSummary.add_summary(summary, step)
        duration = time.time() - start_time
        remain_time=duration*(num_steps-step)/3600
        print('\r',toCyan('{:s}:{:d}-{:d}-{:d} total loss = {:.3f},({:.3f} sec/step, ERT: {:.3f})'.format(args.model_name+'-'+args.domain,step%steps_per_epoch, step//steps_per_epoch,args.num_epochs,total_loss, duration,remain_time)),end='')
        
        if step % args.test_every == 0:
            acc1,acc2=0,0
            for jj in range(val_num_steps):
                feed_dict = {keep_prob: 1}
                src_acc,tar_acc= sess.run([model.src_acc,model.tar_acc], feed_dict=feed_dict)
                acc1+=np.sum(src_acc)
                acc2+=np.sum(tar_acc)

            acc1=acc1/(val_num_steps*args.batch_size)
            acc2=acc2/(val_num_steps*args.batch_size)
            # pdb.set_trace()
            test_summary = tf.Summary()
            test_summary.value.add(tag='test/source_accuracy',simple_value= acc1)
            test_summary.value.add(tag='test/target_accuracy',simple_value= acc2)
            FinalSummary.add_summary(test_summary, step)
            
            if acc2>acc2_history:
                save(saver, sess, args.snapshot_dir, step)
                acc2_history=acc2

    coord.request_stop()
    coord.join(threads)
    sess.close()
Exemplo n.º 45
0
def main():
    # Create model and start training
    h, w = INPUT_SIZE

    X = tf.placeholder(tf.float32, shape=[None, h, w, 3], name='X')
    Y = tf.placeholder(tf.uint8, shape=[None, h, w, 1], name='Y')
    is_training = tf.placeholder(tf.bool, name='is_training')

    net = DeepLabResNetModel(X, is_training, NUM_CLASSES, ATROUS_BLOCKS)

    raw_output = net.output

    # Trainable Variables
    # restore_vars = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
    all_trainable = [
        v for v in tf.trainable_variables()
        if 'beta' not in v.name and 'gamma' not in v.name
    ]
    fc_trainable = [v for v in all_trainable if 'fc' in v.name]
    conv_trainable = [v for v in all_trainable if 'fc' not in v.name]
    fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name]
    fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name]

    # Predictions: ignoring all predictions with labels greater or equal than n_classes
    raw_prediction = tf.reshape(raw_output, [-1, NUM_CLASSES])
    label_proc = prepare_labels(Y,
                                tf.stack(raw_output.get_shape()[1:3]),
                                num_classes=NUM_CLASSES,
                                one_hot=False)
    raw_gt = tf.reshape(label_proc, [
        -1,
    ])
    indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, NUM_CLASSES - 1)), 1)
    gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
    prediction = tf.gather(raw_prediction, indices)

    # Pixel-wise Softmax Loss
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
                                                          labels=gt)
    l2_losses = [
        WEIGHT_DECAY * tf.nn.l2_loss(v) for v in tf.trainable_variables()
        if 'weights' in v.name
    ]
    reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
    variable_summaries(reduced_loss, name='loss')
    variable_summaries(loss, name='loss_origin')

    # Processed predictions: for visualization
    raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(X)[1:3, ])
    raw_output_up = tf.argmax(raw_output_up, dimension=3)
    pred = tf.expand_dims(raw_output_up, dim=3)

    # Define loss and optimization parameters
    base_lr = tf.constant(LEARNING_RATE, tf.float64)
    global_step = tf.Variable(0, trainable=False, name='global_step')
    increment_step = tf.assign(global_step, global_step + 1)
    learning_rate = tf.scalar_mul(base_lr,
                                  tf.pow((1 - global_step / NUM_STEPS), POWER))
    learning_rate = tf.maximum(learning_rate, 8e-7)

    opt_conv = tf.train.MomentumOptimizer(learning_rate, MOMENTUM)
    opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 5.0, MOMENTUM)
    opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 10.0, MOMENTUM)

    grads = tf.gradients(reduced_loss,
                         conv_trainable + fc_w_trainable + fc_b_trainable)
    grads_conv = grads[:len(conv_trainable)]
    grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) +
                                            len(fc_w_trainable))]
    grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]

    train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
    train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
    train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))

    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        train_op = tf.group(increment_step, train_op_conv, train_op_fc_w,
                            train_op_fc_b)

    # initial_learning_rate = 1e-2
    # learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step, 300, 0.96)
    # adam = tf.train.AdamOptimizer(learning_rate).minimize(reduced_loss, global_step=global_step)

    # Image Summary
    images_summary = tf.py_func(inv_preprocess, [X, SAVE_NUM_IMAGES, IMG_MEAN],
                                tf.uint8)
    preds_summary = tf.py_func(decode_labels,
                               [pred, SAVE_NUM_IMAGES, NUM_CLASSES], tf.uint8)
    labels_summary = tf.py_func(decode_labels,
                                [Y, SAVE_NUM_IMAGES, NUM_CLASSES], tf.uint8)

    image_summaries = [images_summary, preds_summary, labels_summary]
    image_summary = tf.summary.image('images',
                                     tf.concat(axis=2, values=image_summaries),
                                     max_outputs=SAVE_NUM_IMAGES)

    # Variable Summary
    variable_summaries(fc_w_trainable, 'fc_w')
    variable_summaries(fc_b_trainable, 'fc_b')
    variable_summaries(learning_rate, 'learning_rate')
    # variable_summaries(net.weights, 'aconv_w')
    # variable_summaries(net.biases, 'aconv_b')

    total_summary = tf.summary.merge_all()

    tb_train_dir = os.path.join(SNAPSHOT_DIR, 'train')
    tb_val_dir = os.path.join(SNAPSHOT_DIR, 'verify')
    summary_writer = tf.summary.FileWriter(tb_train_dir,
                                           graph=tf.get_default_graph())
    verify_writer = tf.summary.FileWriter(tb_val_dir)

    train_data, val_data = read_data()
    train_data_count = len(train_data)
    batch_size = BATCH_SIZE

    print('batch size: %s, total step: %s, train data num: %s' %
          (batch_size, NUM_STEPS, train_data_count))
    # Set up session
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
        saver = tf.train.Saver(max_to_keep=3)
        if SNAPSHOT_DIR is not None and os.path.exists(SNAPSHOT_DIR):
            loader = tf.train.Saver()
            load_model(loader, sess, SNAPSHOT_DIR)

        # export pb
        if False:
            import export_pb
            export_pb.save_graph_with_weight(
                sess, ['ExpandDims'],
                '/data/pb/traffic_line_deeplab_resnet_cut05_271_loss_0_0_93.pb'
            )

        start_index = 0
        for step in range(NUM_STEPS):
            # start_time = time.time()

            if start_index + batch_size > train_data_count:
                start_index = 0
                random.shuffle(train_data)
            _train_data = train_data[start_index:start_index + batch_size]
            start_index += batch_size
            after_processing_data = []
            start_time = time.time()
            for x_img, y_data in _train_data:
                # tmp_img = augmentation(x_img)
                # x_data = np.asarray(tmp_img).astype('float32') / 255.0
                after_processing_data.append((x_img, y_data))
            vec = [d[0] for d in after_processing_data]
            vec2 = [d[1] for d in after_processing_data]
            # data_process_time = time.time() - start_time
            if step % SAVE_SUMMARY_EVERY == 0:
                feed = [
                    reduced_loss, pred, total_summary, global_step, train_op
                ]
                loss_value, preds, summary, total_steps, _ = \
                    sess.run(feed, feed_dict={X: vec, Y: vec2, is_training: True})
                summary_writer.add_summary(summary, total_steps)
            else:
                feed = [reduced_loss, global_step, train_op]
                loss_value, total_steps, _ = sess.run(feed,
                                                      feed_dict={
                                                          X: vec,
                                                          Y: vec2,
                                                          is_training: True
                                                      })
            if step % SAVE_MODEL_EVERY == 0:
                save_model(saver, sess, SNAPSHOT_DIR, global_step)

            duration = time.time() - start_time
            results = 'global step: {:d}, step: {:d} \t loss = {:.3f}, ({:.3f} secs)'\
                .format(total_steps, step, loss_value, duration)
            if step % WRITE_EVERY == 0:
                with open(WRITE_FILE, 'a') as f:
                    f.write(results + '\n')
            print(results)

            if step % VAL_EVERY == 0:
                random.shuffle(val_data)
                vec = [d[0] for d in val_data[:batch_size]]
                vec2 = [d[1] for d in val_data[:batch_size]]
                feed = [reduced_loss, pred, total_summary, global_step]
                loss_value, preds, summary, total_steps = \
                    sess.run(feed, feed_dict={X: vec, Y: vec2, is_training: False})
                verify_writer.add_summary(summary, total_steps)
                print('[Verify]', step, loss_value)
Exemplo n.º 46
0
    def __init__(self,
                 num_classes,
                 vocab_size,
                 shape_domain_size,
                 char_domain_size,
                 char_size,
                 embedding_size,
                 shape_size,
                 nonlinearity,
                 viterbi,
                 hidden_dim,
                 char_embeddings,
                 embeddings=None):

        self.num_classes = num_classes
        self.shape_domain_size = shape_domain_size
        self.char_domain_size = char_domain_size
        self.char_size = char_size
        self.embedding_size = embedding_size
        self.shape_size = shape_size
        self.hidden_dim = hidden_dim
        self.nonlinearity = nonlinearity
        self.char_embeddings = char_embeddings
        self.viterbi = viterbi

        # word embedding input
        self.input_x1 = tf.placeholder(tf.int64, [None, None], name="input_x1")

        # shape embedding input
        self.input_x2 = tf.placeholder(tf.int64, [None, None], name="input_x2")

        # labels
        self.input_y = tf.placeholder(tf.int64, [None, None], name="input_y")

        # padding mask
        self.input_mask = tf.placeholder(tf.float32, [None, None],
                                         name="input_mask")

        self.batch_size = tf.placeholder(tf.int32, None, name="batch_size")

        self.max_seq_len = tf.placeholder(tf.int32, None, name="max_seq_len")

        # sequence lengths
        self.sequence_lengths = tf.placeholder(tf.int32, [None, None],
                                               name="sequence_lengths")

        # dropout and l2 penalties
        self.middle_dropout_keep_prob = tf.placeholder_with_default(
            1.0, [], name="middle_dropout_keep_prob")
        self.hidden_dropout_keep_prob = tf.placeholder_with_default(
            1.0, [], name="hidden_dropout_keep_prob")
        self.input_dropout_keep_prob = tf.placeholder_with_default(
            1.0, [], name="input_dropout_keep_prob")
        self.word_dropout_keep_prob = tf.placeholder_with_default(
            1.0, [], name="word_dropout_keep_prob")

        self.l2_penalty = tf.placeholder_with_default(0.0, [],
                                                      name="l2_penalty")

        self.projection = tf.placeholder_with_default(False, [],
                                                      name="projection")

        self.drop_penalty = tf.placeholder_with_default(0.0, [],
                                                        name="drop_penalty")

        # Keeping track of l2 regularization loss (optional)
        self.l2_loss = tf.constant(0.0)

        # set the pad token to a constant 0 vector
        self.word_zero_pad = tf.constant(0.0,
                                         dtype=tf.float32,
                                         shape=[1, embedding_size])
        self.shape_zero_pad = tf.constant(0.0,
                                          dtype=tf.float32,
                                          shape=[1, shape_size])
        self.char_zero_pad = tf.constant(0.0,
                                         dtype=tf.float32,
                                         shape=[1, char_size])

        self.use_characters = char_size != 0
        self.use_shape = shape_size != 0

        if self.viterbi:
            self.transition_params = tf.get_variable(
                "transitions", [num_classes, num_classes])

        # Embedding layer
        # with tf.device('/cpu:0'), tf.name_scope("embedding"):
        word_embeddings_shape = (vocab_size - 1, embedding_size)
        self.w_e = tf_utils.initialize_embeddings(word_embeddings_shape,
                                                  name="w_e",
                                                  pretrained=embeddings)

        nonzero_elements = tf.not_equal(self.sequence_lengths,
                                        tf.zeros_like(self.sequence_lengths))
        count_nonzero_per_row = tf.reduce_sum(tf.to_int32(nonzero_elements),
                                              axis=1)
        self.flat_sequence_lengths = tf.add(
            tf.reduce_sum(self.sequence_lengths, 1),
            tf.scalar_mul(2, count_nonzero_per_row))

        self.unflat_scores, self.hidden_layer = self.forward(
            self.input_x1,
            self.input_x2,
            self.max_seq_len,
            self.hidden_dropout_keep_prob,
            self.input_dropout_keep_prob,
            self.middle_dropout_keep_prob,
            reuse=False)

        # Calculate mean cross-entropy loss
        with tf.name_scope("loss"):
            labels = tf.cast(self.input_y, 'int32')
            if viterbi:
                log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(
                    self.unflat_scores,
                    labels,
                    self.flat_sequence_lengths,
                    transition_params=self.transition_params)
                # self.transition_params = transition_params
                self.loss = tf.reduce_mean(-log_likelihood)
            else:
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                    logits=self.unflat_scores, labels=labels)
                masked_losses = tf.multiply(losses, self.input_mask)
                self.loss = tf.div(tf.reduce_sum(masked_losses),
                                   tf.reduce_sum(self.input_mask))
            self.loss += self.l2_penalty * self.l2_loss

            self.unflat_no_dropout_scores, _ = self.forward(
                self.input_x1, self.input_x2, self.max_seq_len, 1.0, 1.0, 1.0)

            drop_loss = tf.nn.l2_loss(
                tf.subtract(self.unflat_scores, self.unflat_no_dropout_scores))
            self.loss += self.drop_penalty * drop_loss

        # Accuracy
        with tf.name_scope("predictions"):
            if viterbi:
                self.predictions = self.unflat_scores
            else:
                self.predictions = tf.argmax(self.unflat_scores, 2)
Exemplo n.º 47
0
def train():
    learning_rate = args.lr
    model_path = args.model
    total_epoch = args.epoch
    teacher = nin()
    student = lenet()
    if args.noisy == True:
        drop_scale = 1 / args.Nratio
        noisy_mask = tf.nn.dropout(tf.constant(
            np.float32(np.ones((batch_size, 1))) / drop_scale),
                                   keep_prob=args.Nratio)  #(batchsize,1)
        gaussian = tf.random_normal(shape=[batch_size, 1],
                                    mean=0.0,
                                    stddev=args.Nsigma)
        noisy = tf.mul(noisy_mask, gaussian)
        #noisy_add = tf.add(tf.constant(np.float32(np.ones((batch_size,1)))), noisy)
        teacher = tf.mul(teacher,
                         tf.tile(noisy, tf.constant([1, 10])))  #(batchsize,10)
        #teacher = tf.add(teacher, tf.tile(noisy,tf.constant([1,10])))
        print(bcolors.G + "prepare for training, noisy mode" + bcolors.END)
        tf_loss = tf.nn.l2_loss(teacher - student) / batch_size
    elif args.KD == True:  # correct Hinton method at 2017.1.3
        print(bcolors.G + "prepare for training, knowledge distilling mode" +
              bcolors.END)
        one_hot = tf.one_hot(y, n_classes, 1.0, 0.0)
        #one_hot = tf.cast(one_hot_int, tf.float32)
        teacher_tau = tf.scalar_mul(1.0 / args.tau, teacher)
        student_tau = tf.scalar_mul(1.0 / args.tau, student)
        objective1 = tf.nn.sigmoid_cross_entropy_with_logits(
            student_tau, one_hot)
        objective2 = tf.scalar_mul(0.5, tf.square(student_tau - teacher_tau))
        tf_loss = (args.lamda * tf.reduce_sum(objective1) +
                   (1 - args.lamda) * tf.reduce_sum(objective2)) / batch_size
    else:
        print(bcolors.G + "prepare for training, NIPS2014 mode" + bcolors.END)
        tf_loss = tf.nn.l2_loss(teacher - student) / batch_size

    optimizer1 = tf.train.AdamOptimizer(
        learning_rate=learning_rate).minimize(tf_loss)
    optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate /
                                        10).minimize(tf_loss)

    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
    sess = tf.InteractiveSession(config=tf.ConfigProto(
        gpu_options=gpu_options, allow_soft_placement=True))
    tf.initialize_all_variables().run()
    with tf.device('/cpu:0'):
        saver = tf.train.Saver(max_to_keep=100)
        #saver.restore(sess, os.path.join(model_path,'model-99')
    data, label = read_cifar10('train')
    index = np.array(range(len(data)))  # index randomly ordered
    mean = cal_mean()
    begin = time.time()
    iterations = len(data) / batch_size
    decay_step = int(total_epoch * 0.8)
    cnt = 0
    dropout_rate = args.dropout
    print(bcolors.G + "number of iterations (per epoch) =" +
          str(len(data) / batch_size) + bcolors.END)
    for i in range(total_epoch):
        np.random.shuffle(index)
        cost_sum = 0
        for j in range(iterations):
            batch_x = np.float32(
                data[index[j * batch_size:(j + 1) * batch_size]]) - mean
            batch_y = np.squeeze(
                np.float32(label[index[j * batch_size:(j + 1) * batch_size]]))
            if cnt / decay_step == 0:
                lr = learning_rate
                _, cost = sess.run([optimizer1, tf_loss],
                                   feed_dict={
                                       x: batch_x,
                                       y: batch_y,
                                       keep_prob: 1 - dropout_rate
                                   })
            elif cnt / decay_step == 1:
                lr = learning_rate / 10
                _, cost = sess.run([optimizer2, tf_loss],
                                   feed_dict={
                                       x: batch_x,
                                       y: batch_y,
                                       keep_prob: 1 - dropout_rate
                                   })
            cost_sum += cost
            #pdb.set_trace()
            #if (j % int(iterations*0.25) == 0):
            #    print ("epoch %d-iter %d, cost = %f , avg-cost = %f"%(i, j, cost, cost/n_classes))
            #    sys.stdout.flush()
        cnt += 1
        avg_time = time.time() - begin
        print(
            "epoch %d - avg. %f seconds in each epoch, lr = %.0e, cost = %f , avg-cost-per-logits = %f"
            % (i, avg_time / cnt, lr, cost_sum,
               cost_sum / iterations / n_classes))
        if np.mod(i + 1, 10) == 0:
            print("Epoch ", i + 1, " is done. Saving the model ...")
            with tf.device('/cpu:0'):
                if not os.path.exists(model_path):
                    os.makedirs(model_path)
                saver.save(sess,
                           os.path.join(model_path, 'model'),
                           global_step=i)
        sys.stdout.flush()
Exemplo n.º 48
0
  def loss(self, predictions_dict, groundtruth_dict):
    """Define loss functions given the prediction and the groundtruth dictionary of tensors

    Args:
      predictions_dict: a dictionary holding predicted tensors
      groundtruth_dict: a dictionary holding groundtruth tensors for computing the loss

    Returns:
      loss_dict: a dictionary mapping strings (loss names) to scalar tensors representing
        loss values.
    """
    loss_config = self.model_config.losses
    
    batch_size = self.model_config.batch_size

    batch_reduce_factor = 2
    if self.model_config.input_queue_type in ['source_only', 'target_only']:
      batch_reduce_factor = 1

    loss_dict = {
      'triplet_loss': None,
      'domain_loss': None,
      'classify_loss': None,
      'reconstruction_loss': None,
      'wasserstein_loss': None,
      'kl_loss': None
    }

    if loss_config.reconstruction_loss.use_reconstruction_loss:
      reconstruction_loss = losses.reconstruction_loss(inputs=predictions_dict[PREPROCESSED_INPUTS_KEY],
                                                       outputs=predictions_dict[PRED_RECONSTRUCTION_KEY],
                                                       scale=float(loss_config.reconstruction_loss.reconstruction_loss_weight))

      loss_dict['reconstruction_loss'] = reconstruction_loss

    if loss_config.triplet_loss.use_triplet_loss:
      triplet_loss = losses.triplet_loss(labels=tf.argmax(groundtruth_dict[mnist_dataset_builder.CLASS_LABEL_KEY], axis=1),
                                         embeddings=predictions_dict[EMBEDDING_FEATURES_KEY],
                                         margin=float(loss_config.triplet_loss.triplet_loss_margin),
                                         scale=float(loss_config.triplet_loss.triplet_loss_weight),
                                         batch_reduce_factor=batch_reduce_factor)

      loss_dict['triplet_loss'] = triplet_loss

    if loss_config.domain_loss.use_domain_loss:
      domain_loss, domain_accuracy, _ = losses.softmax_cross_entropy_with_logits_v2(
        logits=predictions_dict[PRED_DOMAIN_KEY],
        labels=groundtruth_dict[
          mnist_dataset_builder.DOMAIN_LABEL_KEY],
        scale=float(
          loss_config.domain_loss.domain_loss_weight), loss_name='Loss/domain_loss')

      loss_dict['domain_loss'] = domain_loss
    
      # TODO(): Add classification_accuracy to the tensorboard summary

    if loss_config.source_classification_loss.use_source_classification_loss:

      source_classify_loss, classify_source_accuracy, classify_target_accuracy = losses.softmax_cross_entropy_with_logits_v2(
        logits=predictions_dict[PRED_CLASSES_KEY], labels=groundtruth_dict[mnist_dataset_builder.CLASS_LABEL_KEY],
        scale=float(loss_config.source_classification_loss.source_classification_loss_weight),
        batch_reduce_factor=batch_reduce_factor, loss_name='Loss/source_classify_loss')

      loss_dict['classify_loss'] = source_classify_loss

    # TODO(): Recheck the output of the wasserstein loss, adjust the batch size
    if loss_config.wasserstein_loss.use_wasserstein_loss:
      wasser_loss = -tf.contrib.gan.losses.wargs.wasserstein_discriminator_loss(
        discriminator_real_outputs=predictions_dict[PRED_DOMAIN_KEY][:batch_size // batch_reduce_factor],
        discriminator_gen_outputs=predictions_dict[PRED_DOMAIN_KEY][batch_size // batch_reduce_factor:])
        
      wasser_loss = tf.scalar_mul(loss_config.wasserstein_loss.wasserstein_loss_weight, wasser_loss)

      loss_dict['wasserstein_loss'] = wasser_loss

    if loss_config.kl_divergence_loss.use_kl_divergence_loss:
      kl_loss = losses.kl_div(predictions_dict[EMBEDDING_FEATURES_KEY][:batch_size // batch_reduce_factor],
                              predictions_dict[EMBEDDING_FEATURES_KEY][batch_size // batch_reduce_factor:])
        
      kl_loss = tf.scalar_mul(loss_config.use_kl_divergence_loss.kl_divergence_loss_weight, kl_loss)

      loss_dict['kl_loss'] = kl_loss

      
    # loss = triplet_loss + domain_loss + reconstruction_loss + classify_loss
    #
    # loss_dict = {
    #   'triplet_loss': triplet_loss,
    #   'domain_loss': domain_loss,
    #   'classify_loss': source_classify_loss,
    #   'reconstruction_loss': reconstruction_loss,
    #   'wasserstein_loss': wasser_loss,
    #   'kl_loss': kl_loss
    # }

    return loss_dict
Exemplo n.º 49
0
def build_network(d):

    # Define hyperparameters
    d = d
    learning_rate = 2e-5
    l2norm_scaling = 1e-10
    global_norm_gradient_clipping_ratio = 0.65

    # Define a placeholder for the answers to the decision problems
    route_exists = tf.placeholder( tf.float32, shape = (None,), name = 'route_exists' )
    # Define a placeholder for the cost of each route
    route_costs = tf.placeholder( tf.float32, shape=(None,1), name='route_costs')
    # Define a placeholder for the edges mask
    edges_mask = tf.placeholder( tf.float32, shape = (None,), name = 'edges_mask' )
    # Define placeholders for the list of number of vertices and edges per instance
    n_vertices  = tf.placeholder( tf.int32, shape = (None,), name = 'n_vertices')
    n_edges     = tf.placeholder( tf.int32, shape = (None,), name = 'edges')
    # 
    EV_matrix   = tf.placeholder( tf.float32, shape = (None,None), name = "EV" )
    edge_weight = tf.placeholder( tf.float32, shape = (None,1), name = "edge_weight" )
    target_cost = tf.placeholder( tf.float32, shape = (None,1), name = "target_cost" )
    time_steps  = tf.placeholder( tf.int32, shape = (), name = "time_steps" )
    
    initial_embedding_mlp = Mlp(
        layer_sizes = [ d for _ in range(3) ],
        activations = [ tf.nn.relu for _ in range(3) ],
        output_size = d,
        name = 'E_init_MLP',
        name_internal_layers = True,
        kernel_initializer = tf.contrib.layers.xavier_initializer(),
        bias_initializer = tf.zeros_initializer()
    )
    
    edge_initial_embeddings = initial_embedding_mlp(
        tf.concat(
          [ edge_weight, target_cost ],
          axis = 1
        )
    )
    
    vertex_initial_embeddings = tf.get_variable(
        initializer = tf.random_normal( (1,d) ),
        dtype = tf.float32, name='V_init'
    )
    total_n = tf.shape( EV_matrix )[1]
    tiled_and_normalized_vertex_initial_embeddings = tf.tile(
        tf.div(
            vertex_initial_embeddings,
            tf.sqrt( tf.cast( total_n, tf.float32 ) )
        ),
        [ total_n, 1 ]
    )

    # Define GNN dictionary
    GNN = {}

    # Define Graph neural network
    gnn = GraphNN(
        {
            # V is the set of vertex embeddings
            'V': d,
            # E is the set of edge embeddings
            'E': d
        },
        {
            # M is a E×V adjacency matrix connecting each edge to the vertices it is connected to
            'EV': ('E','V')
        },
        {
            # V_msg_E is a MLP which computes messages from vertex embeddings to edge embeddings
            'V_msg_E': ('V','E'),
            # E_msg_V is a MLP which computes messages from edge embeddings to vertex embeddings
            'E_msg_V': ('E','V')
        },
        {
            # V(t+1) ← Vu( EVᵀ × E_msg_V(E(t)) )
            'V': [
                {
                    'mat': 'EV',
                    'msg': 'E_msg_V',
                    'transpose?': True,
                    'var': 'E'
                }
            ],
            # E(t+1) ← Eu( EV × V_msg_E(V(t)), W, C )
            'E': [
                {
                    'mat': 'EV',
                    'msg': 'V_msg_E',
                    'var': 'V'
                }
            ]
        },
        name='TSP'
    )

    # Populate GNN dictionary
    GNN['gnn']          = gnn
    GNN['route_exists'] = route_exists
    GNN['route_costs']  = route_costs
    GNN['edges_mask']   = edges_mask
    GNN['n_vertices']   = n_vertices
    GNN['n_edges']      = n_edges
    GNN["EV"]           = EV_matrix
    GNN["W"]            = edge_weight
    GNN["C"]            = target_cost
    GNN["time_steps"]   = time_steps

    # Define E_vote, which will compute one logit for each edge
    E_vote_MLP = Mlp(
        layer_sizes = [ d for _ in range(3) ],
        activations = [ tf.nn.relu for _ in range(3) ],
        output_size = 1,
        name = 'E_vote',
        name_internal_layers = True,
        kernel_initializer = tf.contrib.layers.xavier_initializer(),
        bias_initializer = tf.zeros_initializer()
        )
    vote_bias = tf.get_variable(initializer=tf.zeros_initializer(), shape=(), dtype=tf.float32, name='vote_bias')
    
    # Get the last embeddings
    last_states = gnn(
      { "EV": EV_matrix },
      { "V": tiled_and_normalized_vertex_initial_embeddings, "E": edge_initial_embeddings },
      time_steps = time_steps
    )
    GNN["last_states"] = last_states
    E_n = last_states['E'].h
    # Compute a vote for each embedding
    #E_vote = tf.reshape(E_vote_MLP(tf.concat([E_n,route_costs], axis=1)), [-1])
    E_vote = tf.reshape(E_vote_MLP(E_n), [-1])
    E_prob = tf.sigmoid(E_vote)

    # Compute the number of problems in the batch
    num_problems = tf.shape(n_vertices)[0]
    # n_edges_acc[i] is the number of edges in the batch up to the i-th instance
    n_edges_acc = tf.map_fn(lambda i: tf.reduce_sum(tf.gather(n_edges, tf.range(0,i))), tf.range(0,num_problems))

    # Compute decision predictions (one per problem)
    _, pred_logits = tf.while_loop(
        lambda i, predictions: tf.less(i, num_problems),
        lambda i, predictions:
            (
                (i+1),
                predictions.write(
                    i,
                    #tf.reduce_mean(tf.gather(E_vote, tf.range(n_edges_acc[i], n_edges_acc[i] + n_edges[i])))
                    tf.reduce_mean( E_vote[n_edges_acc[i]:n_edges_acc[i]+n_edges[i]] )
                )
            ),
        [0, tf.TensorArray(size=num_problems, dtype=tf.float32)]
        )
    pred_logits = pred_logits.stack() + vote_bias
    GNN['predictions'] = tf.sigmoid(pred_logits)

    # Count the number of edges that appear in the solution
    pos_edges_n = tf.reduce_sum(edges_mask)
    # Count the number of edges that do not appear in the solution
    neg_edges_n = tf.reduce_sum(tf.subtract(tf.ones_like(edges_mask), edges_mask))
    # Define edges loss
    GNN['loss_edges'] = tf.losses.sigmoid_cross_entropy(
        multi_class_labels  = edges_mask,
        logits              = E_vote,
        weights             = tf.add(
            tf.scalar_mul(
                tf.divide(tf.add(pos_edges_n,neg_edges_n),pos_edges_n),
                edges_mask),
            tf.scalar_mul(
                tf.divide(tf.add(pos_edges_n,neg_edges_n),neg_edges_n),
                tf.subtract(tf.ones_like(edges_mask), edges_mask)
                )
            )
        )
    # Define decision loss
    GNN['loss_decision'] = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=route_exists, logits=pred_logits))

    # Compute true positives, false positives, true negatives, false negatives
    GNN['true_pos']     = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
    GNN['false_pos']    = tf.reduce_sum(tf.multiply(route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
    GNN['true_neg']     = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))
    GNN['false_neg']    = tf.reduce_sum(tf.multiply(1-route_exists, tf.cast(tf.not_equal(route_exists, tf.round(GNN['predictions'])), tf.float32)))

    # Define edges accuracy
    GNN['acc_edges']    = tf.reduce_mean(tf.cast(tf.equal(edges_mask, tf.round(E_prob)), tf.float32))
    # Define decision accuracy
    GNN['acc_decision'] = tf.reduce_mean(tf.cast(tf.equal(route_exists, tf.round(GNN['predictions'])), tf.float32))

    # Define optimizer
    optimizer = tf.train.AdamOptimizer(name='Adam', learning_rate=learning_rate)

    # Compute cost relative to L2 normalization
    vars_cost = tf.add_n([ tf.nn.l2_loss(var) for var in tf.trainable_variables() ])
    
    # Define gradients and train step
    for loss_type in ['edges','decision']:
        grads, _ = tf.clip_by_global_norm(tf.gradients(GNN['loss_' + loss_type] + tf.multiply(vars_cost, l2norm_scaling),tf.trainable_variables()),global_norm_gradient_clipping_ratio)
        GNN['train_step_' + loss_type] = optimizer.apply_gradients(zip(grads, tf.trainable_variables()))
    #end
    
    # Return GNN dictionary
    return GNN
    def _static_subsample(self, indicator, batch_size, labels):
        """Returns subsampled minibatch.

    Args:
      indicator: boolean tensor of shape [N] whose True entries can be sampled.
        N should be a complie time constant.
      batch_size: desired batch size. This scalar cannot be None.
      labels: boolean tensor of shape [N] denoting positive(=True) and negative
        (=False) examples. N should be a complie time constant.

    Returns:
      sampled_idx_indicator: boolean tensor of shape [N], True for entries which
        are sampled. It ensures the length of output of the subsample is always
        batch_size, even when number of examples set to True in indicator is
        less than batch_size.

    Raises:
      ValueError: if labels and indicator are not 1D boolean tensors.
    """
        # Check if indicator and labels have a static size.
        if not indicator.shape.is_fully_defined():
            raise ValueError(
                'indicator must be static in shape when is_static is'
                'True')
        if not labels.shape.is_fully_defined():
            raise ValueError('labels must be static in shape when is_static is'
                             'True')
        if not isinstance(batch_size, int):
            raise ValueError(
                'batch_size has to be an integer when is_static is'
                'True.')

        input_length = tf.shape(indicator)[0]

        # Set the number of examples set True in indicator to be at least
        # batch_size.
        num_true_sampled = tf.reduce_sum(tf.cast(indicator, tf.float32))
        additional_false_sample = tf.less_equal(
            tf.cumsum(tf.cast(tf.logical_not(indicator), tf.float32)),
            batch_size - num_true_sampled)
        indicator = tf.logical_or(indicator, additional_false_sample)

        # Shuffle indicator and label. Need to store the permutation to restore the
        # order post sampling.
        permutation = tf.random_shuffle(tf.range(input_length))
        indicator = ops.matmul_gather_on_zeroth_axis(
            tf.cast(indicator, tf.float32), permutation)
        labels = ops.matmul_gather_on_zeroth_axis(tf.cast(labels, tf.float32),
                                                  permutation)

        # index (starting from 1) when indicator is True, 0 when False
        indicator_idx = tf.where(tf.cast(indicator, tf.bool),
                                 tf.range(1, input_length + 1),
                                 tf.zeros(input_length, tf.int32))

        # Replace -1 for negative, +1 for positive labels
        signed_label = tf.where(
            tf.cast(labels, tf.bool), tf.ones(input_length, tf.int32),
            tf.scalar_mul(-1, tf.ones(input_length, tf.int32)))
        # negative of index for negative label, positive index for positive label,
        # 0 when indicator is False.
        signed_indicator_idx = tf.multiply(indicator_idx, signed_label)
        sorted_signed_indicator_idx = tf.nn.top_k(signed_indicator_idx,
                                                  input_length,
                                                  sorted=True).values

        [num_positive_samples, num_negative_samples
         ] = self._get_num_pos_neg_samples(sorted_signed_indicator_idx,
                                           batch_size)

        sampled_idx = self._get_values_from_start_and_end(
            sorted_signed_indicator_idx, num_positive_samples,
            num_negative_samples, batch_size)

        # Shift the indices to start from 0 and remove any samples that are set as
        # False.
        sampled_idx = tf.abs(sampled_idx) - tf.ones(batch_size, tf.int32)
        sampled_idx = tf.multiply(
            tf.cast(tf.greater_equal(sampled_idx, tf.constant(0)), tf.int32),
            sampled_idx)

        sampled_idx_indicator = tf.cast(
            tf.reduce_sum(tf.one_hot(sampled_idx, depth=input_length), axis=0),
            tf.bool)

        # project back the order based on stored permutations
        reprojections = tf.one_hot(permutation,
                                   depth=input_length,
                                   dtype=tf.float32)
        return tf.cast(
            tf.tensordot(tf.cast(sampled_idx_indicator, tf.float32),
                         reprojections,
                         axes=[0, 0]), tf.bool)
Exemplo n.º 51
0
    def add_prediction_op(self):
        """
        Adds the unrolled RNN:
            h_0 = 0
            for t in 1 to T:
                o_t, h_t = cell(x_t, h_{t-1})
                o_drop_t = Dropout(o_t, dropout_rate)
                y_t = o_drop_t U + b_2

        Returns:
            pred: tf.Tensor of shape (batch_size, max_length, non_terminal_vocab)
        """

        x = self.add_embedding()
        dropout_rate = self.dropout_placeholder

        preds = []  # Predicted output at each timestep should go here!
        hidden = []

        cell = LSTMCell(Config.n_token_features * Config.embed_size,
                        Config.hidden_size)

        # Define U and b2 as variables.
        # Initialize state as vector of zeros.
        xinit = tf.contrib.layers.xavier_initializer(dtype=tf.float64)
        if not self.config.terminal_pred:
            output_size = self.config.non_terminal_vocab
        else:
            output_size = self.config.terminal_vocab

        U = tf.get_variable('U',
                            shape=[self.config.hidden_size, output_size],
                            initializer=xinit,
                            dtype=tf.float64)
        b2 = tf.get_variable('b2',
                             shape=[output_size],
                             initializer=tf.constant_initializer(0.0),
                             dtype=tf.float64)
        c_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size],
                       dtype=tf.float64)
        h_t = tf.zeros([tf.shape(x)[0], self.config.hidden_size],
                       dtype=tf.float64)
        state_tuple = (c_t, h_t)

        scope = "LSTM_terminal" if self.config.terminal_pred else "LSTM_non_terminal"

        with tf.variable_scope(scope):
            for time_step in range(self.max_length):
                if time_step > 0:
                    tf.get_variable_scope().reuse_variables()
                o_t, h_t = cell(x[:, time_step, :], state_tuple)
                o_drop_t = tf.nn.dropout(o_t, dropout_rate)
                preds.append(tf.matmul(o_drop_t, U) + b2)
                hidden.append(h_t[1])

                if not (self.config.cell
                        == "lstmAend") and not (self.config.cell == "lstm"):
                    W_a = tf.get_variable('W_a',
                                          shape=[
                                              self.config.hidden_size,
                                              self.config.hidden_size
                                          ],
                                          dtype=tf.float64,
                                          initializer=xinit)
                    W_o = tf.get_variable(
                        'W_o',
                        shape=[2 * self.config.hidden_size, output_size],
                        dtype=tf.float64,
                        initializer=xinit)
                    W_s = tf.get_variable('W_s',
                                          shape=[output_size, output_size],
                                          dtype=tf.float64,
                                          initializer=xinit)
                    b_o = tf.get_variable(
                        'b_o',
                        shape=[output_size],
                        dtype=tf.float64,
                        initializer=tf.constant_initializer(0.0))
                    b_s = tf.get_variable(
                        'b_s',
                        shape=[output_size],
                        dtype=tf.float64,
                        initializer=tf.constant_initializer(0.0))

                    hidden_stack = tf.stack(hidden, 1)
                    ht = tf.reshape(
                        tf.matmul(h_t[1], W_a),
                        (tf.shape(x)[0], -1, self.config.hidden_size))
                    weights = tf.reduce_sum(
                        ht * hidden_stack, axis=2) * tf.slice(
                            self.attn_mask_placeholder, [0, 0],
                            [-1, time_step + 1])
                    weights = tf.nn.softmax(weights)
                    context = tf.reduce_sum(tf.reshape(
                        weights,
                        (tf.shape(weights)[0], tf.shape(weights)[1], -1)) *
                                            hidden_stack,
                                            axis=1)
                    context_hidden_sum = tf.add(context, h_t[1])

                    if (self.config.cell == "lstmAcont"):
                        hidden = hidden[:-1] + [context]

                    if (self.config.cell == "lstmAsum_fn"):
                        W_alpha = tf.get_variable('W_alpha',
                                                  shape=[
                                                      self.config.hidden_size,
                                                      self.config.hidden_size
                                                  ],
                                                  dtype=tf.float64,
                                                  initializer=xinit)
                        W_beta = tf.get_variable('W_beta',
                                                 shape=[
                                                     self.config.hidden_size,
                                                     self.config.hidden_size
                                                 ],
                                                 dtype=tf.float64,
                                                 initializer=xinit)
                        U_alpha = tf.get_variable('U_alpha',
                                                  shape=[
                                                      self.config.embed_size,
                                                      self.config.hidden_size
                                                  ],
                                                  dtype=tf.float64,
                                                  initializer=xinit)
                        U_beta = tf.get_variable('U_beta',
                                                 shape=[
                                                     self.config.embed_size,
                                                     self.config.hidden_size
                                                 ],
                                                 dtype=tf.float64,
                                                 initializer=xinit)
                        alpha = tf.sigmoid(
                            tf.matmul(context, W_alpha) +
                            tf.matmul(x[:, time_step, :], U_alpha))
                        beta = tf.sigmoid(
                            tf.matmul(h_t[1], W_beta) +
                            tf.matmul(x[:, time_step, :], U_beta))
                        straightSum = alpha * context + beta * h_t[1]
                        hidden = hidden[:-1] + [straightSum]

                    if (self.config.cell == "lstmAsum"):
                        alpha = tf.get_variable(
                            'alpha',
                            shape=(),
                            dtype=tf.float64,
                            initializer=tf.random_uniform_initializer(
                                -1.0, 2.0))
                        beta = tf.get_variable(
                            'beta',
                            shape=(),
                            dtype=tf.float64,
                            initializer=tf.random_uniform_initializer(
                                -1.0, 2.0))
                        straightSum = tf.add(tf.scalar_mul(alpha, context),
                                             tf.scalar_mul(beta, h_t[1]))
                        hidden = hidden[:-1] + [straightSum]

                    if (self.config.cell == "lstmAwsum_fn"):
                        W_ph = tf.get_variable('W_ph',
                                               shape=[
                                                   self.config.hidden_size,
                                                   self.config.hidden_size
                                               ],
                                               dtype=tf.float64,
                                               initializer=xinit)
                        W_pc = tf.get_variable('W_pc',
                                               shape=[
                                                   self.config.hidden_size,
                                                   self.config.hidden_size
                                               ],
                                               dtype=tf.float64,
                                               initializer=xinit)
                        W_px = tf.get_variable('W_px',
                                               shape=[
                                                   self.config.embed_size,
                                                   self.config.hidden_size
                                               ],
                                               dtype=tf.float64,
                                               initializer=xinit)
                        hTerm = tf.matmul(h_t[1], W_ph)
                        cTerm = tf.matmul(context, W_pc)
                        xTerm = tf.matmul(x[:, time_step, :], W_px)
                        p_arr = tf.sigmoid(hTerm + cTerm + xTerm)
                        weightedSum = (p_arr * context) + (
                            (1 - p_arr) * h_t[1])
                        hidden = hidden[:-1] + [weightedSum]

            preds = tf.stack(preds, 1)
            hidden = tf.stack(hidden, 1)
            final_preds = tf.boolean_mask(preds, self.mask_placeholder)
            final_hidden = tf.boolean_mask(hidden, self.mask_placeholder)

        if not (self.config.cell == "lstm"):
            W_a = tf.get_variable(
                'W_a',
                shape=[self.config.hidden_size, self.config.hidden_size],
                dtype=tf.float64,
                initializer=xinit)
            W_o = tf.get_variable(
                'W_o',
                shape=[2 * self.config.hidden_size, output_size],
                dtype=tf.float64,
                initializer=xinit)
            W_s = tf.get_variable('W_s',
                                  shape=[output_size, output_size],
                                  dtype=tf.float64,
                                  initializer=xinit)
            b_o = tf.get_variable('b_o',
                                  shape=[output_size],
                                  dtype=tf.float64,
                                  initializer=tf.constant_initializer(0.0))
            b_s = tf.get_variable('b_s',
                                  shape=[output_size],
                                  dtype=tf.float64,
                                  initializer=tf.constant_initializer(0.0))
            ht = tf.reshape(tf.matmul(final_hidden, W_a),
                            (tf.shape(x)[0], -1, self.config.hidden_size))
            weights = tf.reduce_sum(ht * hidden,
                                    axis=2) * self.attn_mask_placeholder
            weights = tf.nn.softmax(weights)

            context = tf.reduce_sum(
                tf.reshape(weights,
                           (tf.shape(weights)[0], tf.shape(weights)[1], -1)) *
                hidden,
                axis=1)
            final_preds = tf.tanh(
                tf.matmul(tf.concat(1, [context, final_hidden]), W_o) + b_o)
            final_preds = tf.matmul(final_preds, W_s) + b_s

        if self.config.terminal_pred:
            nt = tf.nn.embedding_lookup(
                self.embeddings, self.next_non_terminal_input_placeholder)
            nt = tf.reshape(
                nt,
                [-1, self.config.n_token_features * self.config.embed_size])
            U_nt = tf.get_variable(
                'U_nt',
                shape=[self.config.hidden_size, output_size],
                initializer=xinit,
                dtype=tf.float64)
            b_t = tf.get_variable('b_t',
                                  shape=[output_size],
                                  initializer=tf.constant_initializer(0.0),
                                  dtype=tf.float64)
            final_preds = final_preds + tf.matmul(nt, U_nt) + b_t

        return final_preds
Exemplo n.º 52
0
 def fit(self, mu, x, y_desired):
     y_pred = self(x)
     e = y_desired - y_pred
     x = tf.constant(np.array([1.0] + x, dtype=np.float32))
     self.w.assign_add(tf.scalar_mul(2 * mu * e, x))
Exemplo n.º 53
0
def main():
    """Create the model and start the training."""
    args = get_arguments()
    """
    Get configurations here. We pass some arguments from command line to init configurations, for training hyperparameters, 
    you can set them in TrainConfig Class.

    Note: we set filter scale to 1 for pruned model, 2 for non-pruned model. The filters numbers of non-pruned
          model is two times larger than prunde model, e.g., [h, w, 64] <-> [h, w, 32].
    """
    cfg = TrainConfig(dataset=args.dataset,
                      is_training=True,
                      random_scale=args.random_scale,
                      random_mirror=args.random_mirror,
                      filter_scale=args.filter_scale)
    if args.num_classes is not None:
        cfg.param["num_classes"] = args.num_classes
    if args.data_dir is not None:
        cfg.param["data_dir"] = args.data_dir
    if args.val_list is not None:
        cfg.param["eval_list"] = args.val_list
    if args.train_list is not None:
        cfg.param["train_list"] = args.train_list
    if args.ignore_label is not None:
        cfg.param["ignore_label"] = args.ignore_label
    if args.eval_size is not None:
        cfg.param["eval_size"] = [
            int(x.strip()) for x in args.eval_size.split("x")[::-1]
        ]
    if args.training_size is not None:
        cfg.TRAINING_SIZE = [
            int(x.strip()) for x in args.training_size.split("x")[::-1]
        ]
    if args.batch_size is not None:
        cfg.BATCH_SIZE = args.batch_size
    if args.learning_rate is not None:
        cfg.LEARNING_RATE = args.learning_rate
    if args.restore_from is not None:
        cfg.model_weight = args.restore_from
    if args.snapshot_dir is not None:
        cfg.SNAPSHOT_DIR = args.snapshot_dir
    if args.restore_from == "scratch":
        from tqdm import tqdm
        import cv2
        import joblib as joblib
        if not args.img_mean:
            print(
                "Calculating img mean for custom dataset. To prevent this, specify it with --img-mean next time"
            )
            image_files, annotation_files = read_labeled_image_list(
                cfg.param["data_dir"], cfg.param["train_list"])
            means = joblib.Parallel(n_jobs=6)(
                joblib.delayed(calc_mean)(image_file, cv2)
                for image_file in tqdm(image_files, desc="calc img mean"))
            cfg.IMG_MEAN = np.mean(means, axis=0).tolist()
        else:
            cfg.IMG_MEAN = [float(x.strip()) for x in args.img_mean.split(",")]

    cfg.display()

    # Setup training network and training samples
    train_reader = ImageReader(cfg=cfg, mode='train')
    train_net = ICNet_BN(image_reader=train_reader, cfg=cfg, mode='train')

    loss_sub4, loss_sub24, loss_sub124, reduced_loss = create_losses(
        train_net, train_net.labels, cfg)

    # Setup validation network and validation samples
    with tf.variable_scope('', reuse=True):
        val_reader = ImageReader(cfg, mode='eval')
        val_net = ICNet_BN(image_reader=val_reader, cfg=cfg, mode='train')

        val_loss_sub4, val_loss_sub24, val_loss_sub124, val_reduced_loss = create_losses(
            val_net, val_net.labels, cfg)

    # Using Poly learning rate policy
    base_lr = tf.constant(cfg.LEARNING_RATE)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(
        base_lr, tf.pow((1 - step_ph / cfg.TRAINING_STEPS), cfg.POWER))

    # Set restore variable
    restore_var = tf.global_variables()
    all_trainable = [
        v for v in tf.trainable_variables()
        if ('beta' not in v.name and 'gamma' not in v.name)
        or args.train_beta_gamma
    ]

    # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
    if args.update_mean_var == False:
        update_ops = None
    else:
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    with tf.control_dependencies(update_ops):
        opt_conv = tf.train.MomentumOptimizer(learning_rate, cfg.MOMENTUM)
        grads = tf.gradients(reduced_loss, all_trainable)
        train_op = opt_conv.apply_gradients(zip(grads, all_trainable))

    # Create session & restore weights (Here we only need to use train_net to create session since we reuse it)
    train_net.create_session()
    if args.initializer:
        train_net.set_initializer(initializer_algorithm=args.initializer)
    train_net.initialize_variables()
    if not args.restore_from or args.restore_from != "scratch":
        train_net.restore(cfg.model_weight, restore_var)
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=20)

    total_parameters = 0
    for variable in tf.trainable_variables():
        # shape is an array of tf.Dimension
        shape = variable.get_shape()
        variable_parameters = 1
        for dim in shape:
            variable_parameters *= dim.value
        total_parameters += variable_parameters
    print("Total trainable parameters: " + str(total_parameters))

    # Iterate over training steps.
    val_loss_value = 10.0
    min_val_loss = float("inf")
    stagnation = 0
    max_non_decreasing_val_loss = int(
        np.ceil(args.early_stopping_patience * len(train_reader.image_list) /
                (cfg.BATCH_SIZE * cfg.EVAL_EVERY)))
    print(
        "Maximum times that val loss can stagnate before early stopping is applied: "
        + str(max_non_decreasing_val_loss))
    for step in range(cfg.TRAINING_STEPS):
        start_time = time.time()

        feed_dict = {step_ph: step}
        if step % cfg.EVAL_EVERY == 0:
            loss_value, loss1, loss2, loss3, val_loss_value, _ = train_net.sess.run(
                [
                    reduced_loss, loss_sub4, loss_sub24, loss_sub124,
                    val_reduced_loss, train_op
                ],
                feed_dict=feed_dict)
            if val_loss_value < min_val_loss:
                print("New best val loss {:.3f}. Saving weights...".format(
                    val_loss_value))
                train_net.save(
                    saver,
                    cfg.SNAPSHOT_DIR,
                    step,
                    model_name="val{:.3f}model.ckpt".format(val_loss_value))
                min_val_loss = val_loss_value
                stagnation = 0
            else:
                stagnation += 1
        else:
            loss_value, loss1, loss2, loss3, _ = train_net.sess.run(
                [reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op],
                feed_dict=feed_dict)

        duration = time.time() - start_time
        print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f}, val_loss: {:.3f} ({:.3f} sec/step)'.\
                    format(step, loss_value, loss1, loss2, loss3, val_loss_value, duration))

        if stagnation > max_non_decreasing_val_loss:
            print("Early stopping")
            break
def floatToFixPoint(tensor_):
	tensor_ = tf.clip_by_value(tensor_,-32.0,32.0)
	tensor_ = tf.scalar_mul(67108864.0,tensor_)
	tensor_ = tf.round(tensor_)
	tensor_ = tf.scalar_mul(1/67108864.0,tensor_)
	return (tensor_)
Exemplo n.º 55
0
    def mnist_fid_score(self, epoch, fid_train_images_names):
        def data_reader_faces(filename):
            with tf.device('/CPU'):
                print(tf.cast(filename[0], dtype=tf.string))
                image_string = tf.io.read_file(
                    tf.cast(filename[0], dtype=tf.string))
                # Don't use tf.image.decode_image, or the output shape will be undefined
                image = tf.image.decode_jpeg(image_string, channels=3)
                image.set_shape([218, 178, 3])
                image = tf.image.crop_to_bounding_box(image, 38, 18, 140, 140)
                image = tf.image.resize(image, [80, 80])
                # This will convert to float values in [0, 1]
                image = tf.subtract(image, 127.5)
                image = tf.divide(image, 127.5)
            return image

        self.load_fid_model()

        fid_num_samples = 1000
        fid_batch_size = tf.constant(100, dtype='int64')
        num_parallel_calls = 4

        random_points = tf.keras.backend.random_uniform(
            [min(fid_train_images_names.shape[0], fid_num_samples)],
            minval=0,
            maxval=int(fid_train_images_names.shape[0]),
            dtype='int32',
            seed=None)

        fid_train_images_names = fid_train_images_names[random_points]

        fid_image_dataset = tf.data.Dataset.from_tensor_slices(
            fid_train_images_names)
        fid_image_dataset = fid_image_dataset.map(
            data_reader_faces, num_parallel_calls=int(num_parallel_calls))
        fid_image_dataset = fid_image_dataset.batch(fid_batch_size)

        with tf.device(config.DEVICE):
            count = 0
            fid_sum = 0
            for image_batch in fid_image_dataset:
                noise = tf.random.normal([fid_batch_size, self.noise_dim],
                                         self.noise_mean, self.noise_stddev)
                preds = self.generator(noise, training=False)
                preds = tf.image.resize(preds, [80, 80])
                preds = tf.scalar_mul(2., preds)
                preds = tf.subtract(preds, 1.0)
                preds = preds.numpy()

                act1 = self.fid_model.predict(image_batch)
                act2 = self.fid_model.predict(preds)
                try:
                    act1 = np.concatenate((act1, act1), axis=0)
                    act2 = np.concatenate((act2, act2), axis=0)
                    fid_score = self.calculate_fid(act1, act2)
                    fid_sum += fid_score
                    count += 1
                except:
                    act1 = act1
                    act2 = act2

            avg_fid_score = fid_sum / count / fid_batch_size
            self.fid_scores.append(
                str(epoch) + ',' + str(np.array(avg_fid_score)))
            print("epoch: %d fid: %f" % (epoch, avg_fid_score))
    def train_setup(self):
        tf.set_random_seed(self.conf.random_seed)

        # Create queue coordinator.
        self.coord = tf.train.Coordinator()

        # Input size
        h, w = (self.conf.input_height, self.conf.input_width)
        input_size = (h, w)

        # Devices
        gpu_list = get_available_gpus()
        zip_encoder, zip_decoder_b, zip_decoder_w, zip_crf = [], [], [], []
        previous_crf_names = []
        restore_vars = []
        self.loaders = []

        self.im_list = []

        for i in range(len(gpu_list)):
            with tf.device(gpu_list[i]):
                # Load reader
                with tf.name_scope("create_inputs"):
                    reader = ImageReader(self.conf.data_dir,
                                         self.conf.data_list, input_size,
                                         self.conf.random_scale,
                                         self.conf.random_mirror,
                                         self.conf.ignore_label, IMG_MEAN,
                                         self.coord)
                    self.image_batch, self.label_batch = reader.dequeue(
                        self.conf.batch_size)
                    self.im_list.append(self.image_batch)
                    image_batch_075 = tf.image.resize_images(
                        self.image_batch,
                        [int(h * 0.75), int(w * 0.75)])
                    image_batch_05 = tf.image.resize_images(
                        self.image_batch,
                        [int(h * 0.5), int(w * 0.5)])

                # Create network
                with tf.variable_scope('', reuse=False):
                    net = Deeplab_v2(self.image_batch,
                                     self.conf.num_classes,
                                     True,
                                     rescale075=False,
                                     rescale05=False,
                                     crf_type=self.conf.crf_type)

                with tf.variable_scope('', reuse=True):
                    net075 = Deeplab_v2(image_batch_075,
                                        self.conf.num_classes,
                                        True,
                                        rescale075=True,
                                        rescale05=False,
                                        crf_type=self.conf.crf_type)

                with tf.variable_scope('', reuse=True):
                    net05 = Deeplab_v2(image_batch_05,
                                       self.conf.num_classes,
                                       True,
                                       rescale075=False,
                                       rescale05=True,
                                       crf_type=self.conf.crf_type)

                # Variables that load from pre-trained model.
                restore_var = [
                    v for v in tf.global_variables()
                    if ('fc' not in v.name and 'crfrnn' not in v.name)
                ]
                restore_vars.append(restore_var)

                # Trainable Variables
                all_trainable = tf.trainable_variables()
                # Fine-tune part
                for name in previous_crf_names:
                    for v in all_trainable:
                        if v.name == name:
                            all_trainable.remove(v)

                crf_trainable = [
                    v for v in all_trainable
                    if ('crfrnn' in v.name and v.name not in previous_crf_names
                        )
                ]
                previous_crf_names.extend(v.name for v in crf_trainable)
                encoder_trainable = [
                    v for v in all_trainable
                    if 'fc' not in v.name and 'crfrnn' not in v.name
                ]  # lr * 1.0

                # Remove encoder_trainable from all_trainable
                #all_trainable = [v for v in all_trainable if v not in encoder_trainable]

                # Decoder part
                decoder_trainable = [
                    v for v in all_trainable
                    if 'fc' in v.name and 'crfrnn' not in v.name
                ]

                decoder_w_trainable = [
                    v for v in decoder_trainable
                    if ('weights' in v.name or 'gamma' in v.name)
                    and 'crfrnn' not in v.name
                ]  # lr * 10.0
                decoder_b_trainable = [
                    v for v in decoder_trainable
                    if ('biases' in v.name or 'beta' in v.name)
                    and 'crfrnn' not in v.name
                ]  # lr * 20.0
                # Check
                assert (len(all_trainable) == len(decoder_trainable) +
                        len(crf_trainable)) + len(encoder_trainable)
                assert (len(decoder_trainable) == len(decoder_w_trainable) +
                        len(decoder_b_trainable))

                # Network raw output
                raw_output100 = net.outputs

                raw_output075 = net075.outputs
                raw_output05 = net05.outputs
                raw_output = tf.reduce_max(tf.stack([
                    raw_output100,
                    tf.image.resize_images(raw_output075,
                                           tf.shape(raw_output100)[1:3, ]),
                    tf.image.resize_images(raw_output05,
                                           tf.shape(raw_output100)[1:3, ])
                ]),
                                           axis=0)

                # Ground Truth: ignoring all labels greater or equal than n_classes
                label_proc = prepare_label(self.label_batch,
                                           tf.stack(
                                               raw_output.get_shape()[1:3]),
                                           num_classes=self.conf.num_classes,
                                           one_hot=True)  # [batch_size, h, w]
                label_proc075 = prepare_label(
                    self.label_batch,
                    tf.stack(raw_output075.get_shape()[1:3]),
                    num_classes=self.conf.num_classes,
                    one_hot=True)
                label_proc05 = prepare_label(
                    self.label_batch,
                    tf.stack(raw_output05.get_shape()[1:3]),
                    num_classes=self.conf.num_classes,
                    one_hot=True)

                raw_gt = tf.reshape(label_proc, [
                    -1,
                ])
                raw_gt075 = tf.reshape(label_proc075, [
                    -1,
                ])
                raw_gt05 = tf.reshape(label_proc05, [
                    -1,
                ])

                indices = tf.squeeze(
                    tf.where(tf.less_equal(raw_gt, self.conf.num_classes - 1)),
                    1)
                indices075 = tf.squeeze(
                    tf.where(
                        tf.less_equal(raw_gt075, self.conf.num_classes - 1)),
                    1)
                indices05 = tf.squeeze(
                    tf.where(tf.less_equal(raw_gt05,
                                           self.conf.num_classes - 1)), 1)

                gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
                gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32)
                gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32)

                raw_prediction = tf.reshape(raw_output,
                                            [-1, self.conf.num_classes])
                raw_prediction100 = tf.reshape(raw_output100,
                                               [-1, self.conf.num_classes])
                raw_prediction075 = tf.reshape(raw_output075,
                                               [-1, self.conf.num_classes])
                raw_prediction05 = tf.reshape(raw_output05,
                                              [-1, self.conf.num_classes])

                prediction = tf.gather(raw_prediction, indices)
                prediction100 = tf.gather(raw_prediction100, indices)
                prediction075 = tf.gather(raw_prediction075, indices075)
                prediction05 = tf.gather(raw_prediction05, indices05)

                # Pixel-wise softmax_cross_entropy loss
                #loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
                loss = tf.nn.softmax_cross_entropy_with_logits_v2(
                    logits=raw_prediction,
                    labels=tf.reshape(label_proc[0],
                                      (h * w, self.conf.num_classes)))
                '''
                coefficients = [0.01460247, 1.25147725, 2.88479363, 1.20348121, 1.65261654, 1.67514772,
                                0.62338799, 0.7729363,  0.42038501, 0.98557268, 1.31867536, 0.85313332,
                                0.67227604, 1.21317965, 1.        , 0.24263748, 1.80877607, 1.3082213,
                                0.79664027, 0.72543945, 1.27823374]
                '''
                #loss = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc[0], (h*w, self.conf.num_classes)), logits=raw_prediction)
                #loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction100, labels=gt)
                loss100 = tf.nn.softmax_cross_entropy_with_logits_v2(
                    logits=raw_prediction100,
                    labels=tf.reshape(label_proc[0],
                                      (h * w, self.conf.num_classes)))
                #loss100 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc[0], (h*w, self.conf.num_classes)), logits=raw_prediction100)
                #loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction075, labels=gt075)
                loss075 = tf.nn.softmax_cross_entropy_with_logits_v2(
                    logits=raw_prediction075,
                    labels=tf.reshape(label_proc075[0],
                                      (int(h * 0.75) * int(w * 0.75),
                                       self.conf.num_classes)))
                #loss075 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc075[0], (int(h * 0.75) * int(w * 0.75), self.conf.num_classes)), logits=raw_prediction075)
                #loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction05, labels=gt05)
                loss05 = tf.nn.softmax_cross_entropy_with_logits_v2(
                    logits=raw_prediction05,
                    labels=tf.reshape(
                        label_proc05[0],
                        (int(h * 0.5) * int(w * 0.5), self.conf.num_classes)))
                #loss05 = weighted_loss(self.conf.num_classes, coefficients, labels=tf.reshape(label_proc05[0], (int(h * 0.5) * int(w * 0.5), self.conf.num_classes)), logits=raw_prediction05)

                # L2 regularization
                l2_losses = [
                    self.conf.weight_decay * tf.nn.l2_loss(v)
                    for v in all_trainable if 'weights' in v.name
                ]

                # Loss function
                self.reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean(
                    loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(
                        loss05) + tf.add_n(l2_losses)

                # Define optimizers
                # 'poly' learning rate
                base_lr = tf.constant(self.conf.learning_rate)
                self.curr_step = tf.placeholder(dtype=tf.float32, shape=())
                learning_rate = tf.scalar_mul(
                    base_lr,
                    tf.pow((1 - self.curr_step / self.conf.num_steps),
                           self.conf.power))
                # We have several optimizers here in order to handle the different lr_mult
                # which is a kind of parameters in Caffe. This controls the actual lr for each
                # layer.
                opt_encoder = tf.train.MomentumOptimizer(
                    learning_rate, self.conf.momentum)
                opt_decoder_w = tf.train.MomentumOptimizer(
                    learning_rate * 10.0, self.conf.momentum)
                opt_decoder_b = tf.train.MomentumOptimizer(
                    learning_rate * 20.0, self.conf.momentum)
                opt_crf = tf.train.MomentumOptimizer(learning_rate,
                                                     self.conf.momentum)

                # Gradient accumulation
                # Define a variable to accumulate gradients.
                accum_grads = [
                    tf.Variable(tf.zeros_like(v.initialized_value()),
                                trainable=False) for v in encoder_trainable +
                    decoder_w_trainable + decoder_b_trainable + crf_trainable
                ]

                # Define an operation to clear the accumulated gradients for next batch.
                self.zero_op = [
                    v.assign(tf.zeros_like(v)) for v in accum_grads
                ]
                # To make sure each layer gets updated by different lr's, we do not use 'minimize' here.
                # Instead, we separate the steps compute_grads+update_params.
                # Compute grads
                grads = tf.gradients(
                    self.reduced_loss, encoder_trainable +
                    decoder_w_trainable + decoder_b_trainable + crf_trainable)
                # Accumulate and normalise the gradients.
                self.accum_grads_op = [
                    accum_grads[i].assign_add(grad /
                                              self.conf.grad_update_every)
                    for i, grad in enumerate(grads)
                ]

                grads_encoder = accum_grads[:len(encoder_trainable)]
                grads_decoder_w = accum_grads[len(encoder_trainable
                                                  ):len(encoder_trainable) +
                                              len(decoder_w_trainable)]
                grads_decoder_b = accum_grads[(
                    len(encoder_trainable) +
                    len(decoder_w_trainable)):(len(encoder_trainable) +
                                               len(decoder_w_trainable) +
                                               len(decoder_b_trainable))]
                grads_crf = accum_grads[
                    len(encoder_trainable) + len(decoder_w_trainable) +
                    len(decoder_b_trainable
                        ):]  # assuming crf gradients are appended to the end

                zip_encoder.append(list(zip(grads_encoder, encoder_trainable)))
                zip_decoder_b.append(
                    list(zip(grads_decoder_b, decoder_b_trainable)))
                zip_decoder_w.append(
                    list(zip(grads_decoder_w, decoder_w_trainable)))
                zip_crf.append(list(zip(grads_crf, crf_trainable)))

        avg_grads_encoder = average_gradients(zip_encoder)
        avg_grads_decoder_w = average_gradients(zip_decoder_w)
        avg_grads_decoder_b = average_gradients(zip_decoder_b)
        avg_grads_crf = average_gradients(zip_crf)

        for i in range(len(gpu_list)):
            with tf.device(gpu_list[i]):
                # Update params
                train_op_conv = opt_encoder.apply_gradients(avg_grads_encoder)
                train_op_fc_w = opt_decoder_w.apply_gradients(
                    avg_grads_decoder_w)
                train_op_fc_b = opt_decoder_b.apply_gradients(
                    avg_grads_decoder_b)
                train_op_crf = opt_crf.apply_gradients(avg_grads_crf)

        # Finally, get the train_op!
        update_ops = tf.get_collection(
            tf.GraphKeys.UPDATE_OPS
        )  # for collecting moving_mean and moving_variance
        with tf.control_dependencies(update_ops):
            self.train_op = tf.group(train_op_fc_w, train_op_fc_b,
                                     train_op_crf)  # train_op_conv

        # Saver for storing checkpoints of the model
        self.saver = tf.train.Saver(var_list=tf.global_variables(),
                                    max_to_keep=0)

        # Loader for loading the pre-trained model
        for i in range(len(gpu_list)):
            with tf.device(gpu_list[i]):
                self.loaders.append(tf.train.Saver(var_list=restore_vars[i]))
                #self.loaders.append(tf.train.Saver(var_list=tf.global_variables()))

        # Training summary
        # Processed predictions: for visualisation.
        raw_output_up = tf.image.resize_bilinear(raw_output, input_size)
        raw_output_up = tf.argmax(raw_output_up, axis=3)
        self.pred = tf.expand_dims(raw_output_up, axis=3)
        # Image summary.
        images_summary = tf.py_func(inv_preprocess,
                                    [self.image_batch, 1, IMG_MEAN], tf.uint8)
        labels_summary = tf.py_func(
            decode_labels, [self.label_batch, 1, self.conf.num_classes],
            tf.uint8)
        preds_summary = tf.py_func(decode_labels,
                                   [self.pred, 1, self.conf.num_classes],
                                   tf.uint8)
        self.total_summary = tf.summary.image(
            'images',
            tf.concat(axis=2,
                      values=[images_summary, labels_summary, preds_summary]),
            max_outputs=1)  # Concatenate row-wise.
        if not os.path.exists(self.conf.logdir):
            os.makedirs(self.conf.logdir)
        self.summary_writer = tf.summary.FileWriter(
            self.conf.logdir, graph=tf.get_default_graph())
def preprocess_for_train(image, classes, boxes, resolution, speed_mode=False):
    if speed_mode:
        pass
    else:

        image_ori = image
        classes_ori = classes
        boxes_ori = boxes

        # randomly sample patches
        if RANDOM_SUB_SAMPLE:
            image_shape = tf.shape(image)

            x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1)
            x1 = tf.expand_dims(x1, -1)
            x2 = tf.expand_dims(x2, -1)
            y1 = tf.expand_dims(y1, -1)
            y2 = tf.expand_dims(y2, -1)
            boxes = tf.concat([y1, x1, y2, x2], axis=1)

            bbox_begin, bbox_size, distort_bbox = tf.image.sample_distorted_bounding_box(
                image_shape,
                bounding_boxes=tf.expand_dims(boxes, 0),
                min_object_covered=BBOX_CROP_OVERLAP,
                aspect_ratio_range=(0.5, 2.0),
                area_range=(0.3, 1.0),
                max_attempts=200,
                use_image_if_no_bounding_boxes=True)

            distort_bbox = distort_bbox[0, 0]

            image = tf.slice(image, bbox_begin, bbox_size)

            boxes = bboxes_resize(distort_bbox, boxes)

            classes, boxes = bboxes_filter_overlap(classes,
                                                   boxes,
                                                   threshold=BBOX_CROP_OVERLAP,
                                                   assign_negative=False)

            y1, x1, y2, x2 = tf.unstack(boxes, 4, axis=1)
            image_shape = tf.shape(image)

            x1 = tf.clip_by_value(x1, 0.0, 1.0)
            x2 = tf.clip_by_value(x2, 0.0, 1.0)
            y1 = tf.clip_by_value(y1, 0.0, 1.0)
            y2 = tf.clip_by_value(y2, 0.0, 1.0)

            x1 = tf.expand_dims(x1, -1)
            x2 = tf.expand_dims(x2, -1)
            y1 = tf.expand_dims(y1, -1)
            y2 = tf.expand_dims(y2, -1)

            boxes = tf.concat([x1, y1, x2, y2], axis=1)

        if RANDOM_ZOOM_OUT:
            image, boxes = random_zoom_out(image, boxes)

        if RANDOM_HFLIP:
            # Randomly flip the image horizontally.
            image, boxes = random_flip_left_right(image, boxes)

        if RANDOM_COLOR:
            # Color distortion
            image = tf.div(image, 255.0)
            image = apply_with_random_selector(
                image,
                lambda x, ordering: distort_color(x, ordering, fast_mode=False
                                                  ),
                num_cases=4)
            image = tf.scalar_mul(255.0, image)

        # Rollback to the full image if there is no boxes
        no_box_cond = tf.equal(tf.size(boxes), 0)
        image = tf.cond(no_box_cond, lambda: image_ori, lambda: image)
        boxes = tf.cond(no_box_cond, lambda: boxes_ori, lambda: boxes)
        classes = tf.cond(no_box_cond, lambda: classes_ori, lambda: classes)

        # Scaling to canonical size without perspective preserved
        image, scale, translation = aspect_preserving_resize(
            image, resolution, depth=3, resize_mode="bilinear")
        new_image = tf.image.resize_image_with_crop_or_pad(
            image, resolution, resolution)

        scale_x = tf.to_float(tf.shape(image)[1]) / tf.to_float(resolution)
        scale_y = tf.to_float(tf.shape(image)[0]) / tf.to_float(resolution)
        shift_x = tf.math.maximum(
            0.0,
            tf.to_float(resolution - tf.shape(image)[1]) /
            float(resolution * 2))
        shift_y = tf.math.maximum(
            0.0,
            tf.to_float(resolution - tf.shape(image)[0]) /
            float(resolution * 2))
        x1, y1, x2, y2 = tf.unstack(boxes, 4, axis=1)
        x1 = tf.scalar_mul(scale_x, x1)
        y1 = tf.scalar_mul(scale_y, y1)
        x2 = tf.scalar_mul(scale_x, x2)
        y2 = tf.scalar_mul(scale_y, y2)
        boxes = tf.concat([
            tf.expand_dims(x1, -1),
            tf.expand_dims(y1, -1),
            tf.expand_dims(x2, -1),
            tf.expand_dims(y2, -1)
        ],
                          axis=1)
        boxes = boxes + [shift_x, shift_y, shift_x, shift_y]
        image = new_image

        # Scaling to canonical size without preserving perspective
        # image, scale, translation = bilinear_resize(image, resolution, depth=3, resize_mode="bilinear")

        # mean subtraction
        means = [_R_MEAN, _G_MEAN, _B_MEAN]
        channels = tf.split(axis=2, num_or_size_splits=3, value=image)
        for i in range(3):
            channels[i] -= means[i]
        # image = tf.concat(axis=2, values=channels)

        # caffe swaps color channels
        image = tf.concat(axis=2,
                          values=[channels[2], channels[1], channels[0]])

    return image, classes, boxes, scale, translation
Exemplo n.º 58
0
def main():
    
    """Create the model and start the training."""
    args = get_arguments()
    
    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)
    
    coord = tf.train.Coordinator()
    
    with tf.name_scope("create_inputs"):
        reader = ImageReader(
            DATA_DIR,
            DATA_LIST_PATH,
            input_size,
            args.random_scale,
            args.random_mirror,
            args.ignore_label,
            IMG_MEAN,
            coord)
        image_batch, label_batch = reader.dequeue(args.batch_size)
    
    net = ICNet_BN({'data': image_batch}, is_training=True, num_classes=args.num_classes, filter_scale=args.filter_scale)
    
    sub4_out = net.layers['sub4_out']
    sub24_out = net.layers['sub24_out']
    sub124_out = net.layers['conv6_cls']

    restore_var = tf.global_variables()
    # restore_var = [v for v in tf.global_variables() if 'conv6_cls' not in v.name]
    all_trainable = [v for v in tf.trainable_variables() if ('beta' not in v.name and 'gamma' not in v.name) or args.train_beta_gamma]
   
    loss_sub4 = create_loss(sub4_out, label_batch, args.num_classes, args.ignore_label)
    loss_sub24 = create_loss(sub24_out, label_batch, args.num_classes, args.ignore_label)
    loss_sub124 = create_loss(sub124_out, label_batch, args.num_classes, args.ignore_label)
    l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
    
    reduced_loss = LAMBDA1 * loss_sub4 +  LAMBDA2 * loss_sub24 + LAMBDA3 * loss_sub124 + tf.add_n(l2_losses)

    # Using Poly learning rate policy 
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
    
    # Gets moving_mean and moving_variance update operations from tf.GraphKeys.UPDATE_OPS
    if args.update_mean_var == False:
        update_ops = None
    else:
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    with tf.control_dependencies(update_ops):
        opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
        grads = tf.gradients(reduced_loss, all_trainable)
        train_op = opt_conv.apply_gradients(zip(grads, all_trainable))
        
    # Set up tf session and initialize variables. 
    # 先初始化所有变量
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()
    
    sess.run(init)
    
    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=5)

    ckpt = tf.train.get_checkpoint_state(args.snapshot_dir)
    if ckpt and ckpt.model_checkpoint_path:
        loader = tf.train.Saver(var_list=restore_var)
        load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1])
        load(loader, sess, ckpt.model_checkpoint_path)
    else:
        print('Restore from pre-trained model...')
        net.load(args.restore_from, sess)
        # args.restore_from:./model/icnet_cityscapes_trainval_90k_bnnomerge.npy
        # net_data = np.load(open(args.restore_from,"rb"),encoding="latin1").item()
        # loader = tf.train.Saver(var_list=exclude_restore_var)
        # loader.restore(sess, net_data)
        
                

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        
        feed_dict = {step_ph: step}
        if step % args.save_pred_every == 0:
            loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
            save(saver, sess, args.snapshot_dir, step)
        else:
            loss_value, loss1, loss2, loss3, _ = sess.run([reduced_loss, loss_sub4, loss_sub24, loss_sub124, train_op], feed_dict=feed_dict)
        duration = time.time() - start_time
        print('step {:d} \t total loss = {:.3f}, sub4 = {:.3f}, sub24 = {:.3f}, sub124 = {:.3f} ({:.3f} sec/step)'.format(step, loss_value, loss1, loss2, loss3, duration))
        
    coord.request_stop()
    coord.join(threads)
Exemplo n.º 59
0
for i in range(args.train_batch):
    fake_labels[i, tmp_fake_labels[i]] = 1

loss = categorical_crossentropy(net.y_, net.y)
top1 = categorical_accuracy(net.y_, net.y)
top5 = top_k_categorical_accuracy(net.y_, net.y, 5)

base_lr = 0.02
step = tf.Variable(0, trainable=False, name="Step")
learning_rate = tf.train.exponential_decay(base_lr, step, 1, 0.999964)

weight_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "W:0"]
bias_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "b:0"]

optimizer1 = tf.train.MomentumOptimizer(learning_rate, 0.9)
optimizer2 = tf.train.MomentumOptimizer(tf.scalar_mul(2.0, learning_rate), 0.9)
grads = optimizer1.compute_gradients(loss, var_list=weight_list+bias_list)
w_grads = grads[:len(weight_list)]
b_grads = grads[len(weight_list):]

train1 = optimizer1.apply_gradients(w_grads, global_step=step)
train2 = optimizer2.apply_gradients(b_grads, global_step=step)
train_step = tf.group(train1, train2)

init = tf.global_variables_initializer()
sess = tf.Session()
K.set_session(sess)

sess.run(init)

start = time.time()
Exemplo n.º 60
0
dataset = tf.data.Dataset.from_tensor_slices((imNames, imLabels, imSplit))
dataset = dataset.map(parseFunction,
                      num_parallel_calls=options.numParallelLoaders)
dataset = dataset.shuffle(buffer_size=numItemsInDataset, seed=0)
dataset = dataset.batch(options.batchSize)

iterator = dataset.make_initializable_iterator()

with tf.name_scope('Model'):
    # Data placeholders
    inputBatchImages, inputBatchImageNames, inputBatchImageLabels, inputBatchImageSplit = iterator.get_next(
    )
    print("Data shape: %s" % str(inputBatchImages.get_shape()))

    if (options.modelName == "IncResV2") or (options.modelName == "NASNet"):
        scaledInputBatchImages = tf.scalar_mul((1.0 / 255), inputBatchImages)
        scaledInputBatchImages = tf.subtract(scaledInputBatchImages, 0.5)
        scaledInputBatchImages = tf.multiply(scaledInputBatchImages, 2.0)

        # Create model
        if options.modelName == "IncResV2":
            arg_scope = inception_resnet_v2.inception_resnet_v2_arg_scope()
            with slim.arg_scope(arg_scope):
                logits, aux_logits, endPoints = inception_resnet_v2.inception_resnet_v2(
                    scaledInputBatchImages, is_training=False)

                # Get the lower layer and upper layer activations
                lowerLayerActivations = endPoints["?"]
                upperLayerActivations = endPoints["?"]

            # Create list of vars to restore before train op