def testMapSparseTensor(self):
     with self.test_session():
         with self.assertRaises(TypeError):
             tf.map_fn(
                 lambda x: x,
                 tf.SparseTensor(indices=[[0, 0], [0, 1], [1, 0]], values=tf.constant([0, 1, 2]), shape=[2, 2]),
             )
  def testMap_Scoped(self):
    with self.test_session() as sess:

      def double_scoped(x):
        """2x with a dummy 2 that is scoped."""
        with tf.variable_scope("body"):
          # Dummy variable, just to check that scoping works as intended.
          two = tf.get_variable("two", [], dtype=tf.int32,
                                initializer=tf.constant_initializer(2))
          return tf.mul(x, two)

      with tf.variable_scope("root") as varscope:
        elems = tf.constant([1, 2, 3, 4, 5, 6], name="data")
        doubles = np.array([2*x for x in [1, 2, 3, 4, 5, 6]])

        r = tf.map_fn(double_scoped, elems)
        # Check that we have the one variable we asked for here.
        self.assertEqual(len(tf.trainable_variables()), 1)
        self.assertEqual(tf.trainable_variables()[0].name, "root/body/two:0")
        sess.run([tf.initialize_all_variables()])
        self.assertAllEqual(doubles, r.eval())

        # Now let's reuse our single variable.
        varscope.reuse_variables()
        r = tf.map_fn(double_scoped, elems)
        self.assertEqual(len(tf.trainable_variables()), 1)
        self.assertAllEqual(doubles, r.eval())
def alexnet(image, batch_size=50):
    """ Canonical alexnet implementation. See the network detail above
    Arg:
        images: pixels value in an array (transformed to a tensor)

    Return:
        logits
    """
    x_image = tf.reshape(image, [-1, 32, 32, 3])

    # Randomly crop a [height, width] section of the image.
    # distorted_image = tf.random_crop(x_image, [height, width, 3])

    # Randomly flip the image horizontally.
    # distorted_image = tf.image.random_flip_left_right(x_image)
    distorted_image = tf.map_fn(lambda image: tf.image.random_flip_left_right(image), x_image)

    # Because these operations are not commutative, consider randomizing
    # the order their operation.
    # NOTE: since per_image_standardization zeros the mean and makes
    # the stddev unit, this likely has no effect see tensorflow#1458.
    distorted_image = tf.map_fn(lambda image: tf.image.random_brightness(image, max_delta=63), distorted_image)
    distorted_image = tf.map_fn(lambda image: tf.image.random_contrast(image, lower=0.2, upper=1.8), distorted_image)

    # Subtract off the mean and divide by the variance of the pixels.
    float_image = tf.map_fn(lambda image: tf.image.per_image_standardization(image), distorted_image)

    # conv1
    conv1 = conv_layer(float_image, 3, 64, "conv1")

    #poo1
    pool1 = tf.nn.max_pool(conv1, ksize=[1,3,3,1], strides=[1,2,2,1], padding='SAME')

    #norm1
    norm1 = tf.nn.lrn(pool1, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)

    #conv2
    W_conv2 = weight_variable([5,5,64,64])
    conv2 = conv_layer(norm1, 64, 64, "conv2")

    #norm2
    norm2 = tf.nn.lrn(conv2, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)

    #poo2
    pool2 = tf.nn.max_pool(norm2, ksize=[1,3,3,1], strides=[1,2,2,1], padding='SAME')

    #fc1, fully connected layer
    reshape = tf.reshape(pool2, [-1, 8 * 8 * 64])
    fc1 = fc_layer(reshape, 8*8*64, 1024, "fc1")

    #local4
    # weights = weight_variable([384, 192])
    # biases = bias_variable([192])
    # local4 = tf.nn.relu(tf.matmul(local3, weights) + biases)


    # linear layer(WX + b),
    logits = fc_layer(fc1, 1024, 10, "output_layer", act=tf.identity)

    return logits
Esempio n. 4
0
def parse_sequence_to_pairs_batch(
    serialized_example, preprocess_fn, is_training, num_views, batch_size,
    window):
  """Parses a serialized sequence example into a batch of preprocessed data.

  Args:
    serialized_example: A serialized SequenceExample.
    preprocess_fn: A function with the signature (raw_images, is_training) ->
      preprocessed_images.
    is_training: Boolean, whether or not we're in training.
    num_views: Int, the number of simultaneous viewpoints at each timestep in
      the dataset.
    batch_size: Int, size of the batch to get.
    window: Int, only take pairs from a maximium window of this size.
  Returns:
    preprocessed: A 4-D float32 `Tensor` holding preprocessed images.
    anchor_images: A 4-D float32 `Tensor` holding raw anchor images.
    pos_images: A 4-D float32 `Tensor` holding raw positive images.
  """
  _, views, seq_len = parse_sequence_example(serialized_example, num_views)

  # Get random (anchor, positive) timestep and viewpoint indices.
  num_pairs = batch_size // 2
  ap_time_indices, a_view_indices, p_view_indices = get_tcn_anchor_pos_indices(
      seq_len, num_views, num_pairs, window)

  # Gather the image strings.
  combined_anchor_indices = tf.concat(
      [tf.expand_dims(a_view_indices, 1),
       tf.expand_dims(ap_time_indices, 1)], 1)
  combined_pos_indices = tf.concat(
      [tf.expand_dims(p_view_indices, 1),
       tf.expand_dims(ap_time_indices, 1)], 1)
  anchor_images = tf.gather_nd(views, combined_anchor_indices)
  pos_images = tf.gather_nd(views, combined_pos_indices)

  # Decode images.
  anchor_images = tf.map_fn(
      preprocessing.decode_image, anchor_images, dtype=tf.float32)
  pos_images = tf.map_fn(
      preprocessing.decode_image, pos_images, dtype=tf.float32)

  # Concatenate [anchor, postitive] images into a batch and preprocess it.
  concatenated = tf.concat([anchor_images, pos_images], 0)
  preprocessed = preprocess_fn(concatenated, is_training)
  anchor_prepro, positive_prepro = tf.split(preprocessed, num_or_size_splits=2,
                                            axis=0)

  # Set static batch dimensions for all image tensors
  ims = [anchor_prepro, positive_prepro, anchor_images, pos_images]
  ims = [set_image_tensor_batch_dim(i, num_pairs) for i in ims]
  [anchor_prepro, positive_prepro, anchor_images, pos_images] = ims

  # Assign each anchor and positive the same label.
  anchor_labels = tf.range(1, num_pairs+1)
  positive_labels = tf.range(1, num_pairs+1)

  return (anchor_prepro, positive_prepro, anchor_images, pos_images,
          anchor_labels, positive_labels, seq_len)
 def testMap_MultiOutputMismatchedDtype(self):
   with self.test_session():
     nums = np.array([1, 2, 3, 4, 5, 6])
     with self.assertRaisesRegexp(
         TypeError, r"two structures don't have the same sequence type."):
       # lambda emits tuple, but dtype is a list
       tf.map_fn(lambda x: ((x + 3) * 2, -(x + 3) * 2), nums,
                 dtype=[tf.int64, tf.int64])
Esempio n. 6
0
    def loss(self, logits, labels, weights=None, biases=None):
        with tf.name_scope("xent"):
            loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels))

            l2_reg = 0.0
            if weights:
                l2_reg += sum(tf.map_fn(tf.nn.l2_loss, weights))
            if biases:
                l2_reg += sum(tf.map_fn(tf.nn.l2_loss, biases))
            loss += self.l2_lambda * l2_reg
        return loss
Esempio n. 7
0
def compare_vs_map(X1, X2, kern, sess):
    K12_map = tf.map_fn(lambda x: kern.K(x[0], x[1]), [X1, X2], dtype=settings.float_type)
    K12_native = kern.K(X1, X2)
    assert_allclose(*sess.run([K12_map, K12_native]))

    K_map = tf.map_fn(kern.K, X1, dtype=settings.float_type)
    K_native = kern.K(X1)
    assert_allclose(*sess.run([K_map, K_native]))

    Kdiag_map = tf.map_fn(kern.Kdiag, X1, dtype=settings.float_type)
    Kdiag_native = kern.Kdiag(X1)
    assert_allclose(*sess.run([Kdiag_map, Kdiag_native]))
Esempio n. 8
0
 def __call__(self, *parents):
   if len(parents) != 1:
     raise ValueError("Must have one parent")
   parent_tensor = parents[0].out_tensor
   dense_fn = lambda x: tf.contrib.layers.fully_connected(x, num_outputs=self.out_channels,
                                                          activation_fn=tf.nn.sigmoid)
   self.out_tensor = tf.map_fn(dense_fn, parent_tensor)
Esempio n. 9
0
def preprocess_training_images(images, height, width, min_scale, max_scale,
                               p_scale_up, aug_color=True, fast_mode=True):
  """Preprocesses a batch of images for training.

  This applies training-time scale and color augmentation, crops/resizes,
  and scales images to the [-1,1] range expected by pre-trained Inception nets.

  Args:
    images: A 4-D float32 `Tensor` holding raw images to be preprocessed.
    height: Int, height in pixels to resize image to.
    width: Int, width in pixels to resize image to.
    min_scale: Float, minimum scale augmentation allowed, as a fraction of the
      central min_side * min_side area of the original image.
    max_scale: Float, maximum scale augmentation allowed, as a fraction of the
      central min_side * min_side area of the original image.
    p_scale_up: Float, fraction of images scaled up.
    aug_color: Whether or not to do color augmentation.
    fast_mode: Boolean, avoids slower ops (random_hue and random_contrast).
  Returns:
    preprocessed_images: A 4-D float32 `Tensor` holding preprocessed images.
  """
  def _prepro_train(im):
    """Map this preprocessing function over each image in the batch."""
    return preprocess_training_image(
        im, height, width, min_scale, max_scale, p_scale_up,
        aug_color=aug_color, fast_mode=fast_mode)
  return tf.map_fn(_prepro_train, images)
Esempio n. 10
0
def bboxes_sort_all_classes(classes, scores, bboxes, top_k=400, scope=None):
    """Sort bounding boxes by decreasing order and keep only the top_k.
    Assume the input Tensors mix-up objects with different classes.
    Assume a batch-type input.

    Args:
      classes: Batch x N Tensor containing integer classes.
      scores: Batch x N Tensor containing float scores.
      bboxes: Batch x N x 4 Tensor containing boxes coordinates.
      top_k: Top_k boxes to keep.
    Return:
      classes, scores, bboxes: Sorted tensors of shape Batch x Top_k.
    """
    with tf.name_scope(scope, 'bboxes_sort', [classes, scores, bboxes]):
        scores, idxes = tf.nn.top_k(scores, k=top_k, sorted=True)

        # Trick to be able to use tf.gather: map for each element in the batch.
        def fn_gather(classes, bboxes, idxes):
            cl = tf.gather(classes, idxes)
            bb = tf.gather(bboxes, idxes)
            return [cl, bb]
        r = tf.map_fn(lambda x: fn_gather(x[0], x[1], x[2]),
                      [classes, bboxes, idxes],
                      dtype=[classes.dtype, bboxes.dtype],
                      parallel_iterations=10,
                      back_prop=False,
                      swap_memory=False,
                      infer_shape=True)
        classes = r[0]
        bboxes = r[1]
        return classes, scores, bboxes
 def input_fn():
   starts = tf.random_uniform([batch_size], maxval=(2 * np.pi), seed=seed)
   sin_curves = tf.map_fn(_sin_fn, (starts,), dtype=tf.float32)
   inputs = tf.expand_dims(
       tf.slice(sin_curves, [0, 0], [batch_size, sequence_length]), 2)
   labels = tf.slice(sin_curves, [0, 1], [batch_size, sequence_length])
   return {'inputs': inputs}, labels
Esempio n. 12
0
  def _decode_png_instance_masks(self, keys_to_tensors):
    """Decode PNG instance segmentation masks and stack into dense tensor.

    The instance segmentation masks are reshaped to [num_instances, height,
    width].

    Args:
      keys_to_tensors: a dictionary from keys to tensors.

    Returns:
      A 3-D float tensor of shape [num_instances, height, width] with values
        in {0, 1}.
    """

    def decode_png_mask(image_buffer):
      image = tf.squeeze(
          tf.image.decode_image(image_buffer, channels=1), axis=2)
      image.set_shape([None, None])
      image = tf.to_float(tf.greater(image, 0))
      return image

    png_masks = keys_to_tensors['image/object/mask']
    height = keys_to_tensors['image/height']
    width = keys_to_tensors['image/width']
    if isinstance(png_masks, tf.SparseTensor):
      png_masks = tf.sparse_tensor_to_dense(png_masks, default_value='')
    return tf.cond(
        tf.greater(tf.size(png_masks), 0),
        lambda: tf.map_fn(decode_png_mask, png_masks, dtype=tf.float32),
        lambda: tf.zeros(tf.to_int32(tf.stack([0, height, width]))))
Esempio n. 13
0
def _evaluate_objective_multiple(objective_function, arg_batch,
                                 batch_evaluate_objective):
  """Evaluates the objective function on a batch of points.

  If `batch_evaluate_objective` is True, returns
  `objective function(arg_batch)` else it maps the `objective_function`
  across the `arg_batch`.

  Args:
    objective_function: A Python callable that accepts a single `Tensor` of
      rank 'R > 1' and any shape 's' and returns a scalar `Tensor` of real dtype
      containing the value of the function at that point. If
      `batch a `Tensor` of shape `[batch_size] + s ` where `batch_size` is the
      size of the batch of args. In this case, the expected return value is a
      `Tensor` of shape `[batch_size]`.
    arg_batch: A `Tensor` of real dtype. The batch of arguments at which to
      evaluate the `objective_function`. If `batch_evaluate_objective` is False,
      `arg_batch` will be unpacked along the zeroth axis and the
      `objective_function` will be applied to each element.
    batch_evaluate_objective: `bool`. Whether the `objective_function` can
      evaluate a batch of arguments at once.

  Returns:
    A tuple containing:
      objective_values: A `Tensor` of real dtype and shape `[batch_size]`.
        The value of the objective function evaluated at the supplied
        `arg_batch`.
      num_evaluations: An `int32` scalar `Tensor`containing the number of
        points on which the objective function was evaluated (i.e `batch_size`).
  """
  n_points = tf.shape(arg_batch)[0]
  if batch_evaluate_objective:
    return objective_function(arg_batch), n_points
  return tf.map_fn(objective_function, arg_batch), n_points
Esempio n. 14
0
def normalized_to_image_coordinates(normalized_boxes, image_shape,
                                    parallel_iterations=32):
    """Converts a batch of boxes from normal to image coordinates.

    Args:
      normalized_boxes: a float32 tensor of shape [None, num_boxes, 4] in
        normalized coordinates.
      image_shape: a float32 tensor of shape [4] containing the image shape.
      parallel_iterations: parallelism for the map_fn op.

    Returns:
      absolute_boxes: a float32 tensor of shape [None, num_boxes, 4] containg the
        boxes in image coordinates.
    """

    def _to_absolute_coordinates(normalized_boxes):
        return box_list_ops.to_absolute_coordinates(
            box_list.BoxList(normalized_boxes),
            image_shape[1], image_shape[2], check_range=False).get()

    absolute_boxes = tf.map_fn(
        _to_absolute_coordinates,
        elems=(normalized_boxes),
        dtype=tf.float32,
        parallel_iterations=parallel_iterations,
        back_prop=True)
    return absolute_boxes
Esempio n. 15
0
def standardize(images):
    """Linearly scales `image` to have zero mean and unit norm.
    (A mirror to tf.image per_image_standardization)

    This op computes `(x - mean) / adjusted_stddev`, where `mean` is the average
    of all values in image, and
    `adjusted_stddev = max(stddev, 1.0/sqrt(image.NumElements()))`.

    `stddev` is the standard deviation of all values in `image`. It is capped
    away from zero to protect against division by 0 when handling uniform images.

    Args:
        images: 4-D Tensor of shape `[batch, height, width, channels]` or
                3-D Tensor of shape `[height, width, channels]`.

    Returns:
        The standardized image with same shape as `image`.

    Raises:
        ValueError: if the shape of 'image' is incompatible with this function.

    """
    images_shape = get_shape(images)
    if len(images_shape) > 4:
        ValueError("'image' must have either 3 or 4 dimensions, "
                   "received `{}`.".format(images_shape))

    if len(images_shape) == 4:
        return tf.map_fn(lambda img: tf.image.per_image_standardization(img), images)
    return tf.image.per_image_standardization(images)
Esempio n. 16
0
  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    inputs = self._get_input_tensors(in_layers)
    parent_tensor = inputs[0]
    training = kwargs['training'] if 'training' in kwargs else 1.0

    parent_tensor = parent_tensor / 255.0
    if not self.augment:
      out_tensor = parent_tensor
    else:

      def preprocess(img):
        img = tf.image.random_flip_left_right(img)
        img = tf.image.random_flip_up_down(img)
        img = tf.image.rot90(img, k=np.random.randint(0, 4))
        if self.distort_color:
          img = tf.image.random_brightness(img, max_delta=32. / 255.)
          img = tf.image.random_saturation(img, lower=0.5, upper=1.5)
          img = tf.clip_by_value(img, 0.0, 1.0)
        if self.central_crop:
          # sample cut ratio from a clipped gaussian
          img = tf.image.central_crop(img,
                                      np.clip(
                                          np.random.normal(1., 0.06), 0.8, 1.))
          img = tf.image.resize_bilinear(
              tf.expand_dims(img, 0), tf.convert_to_tensor(self.size))[0]
        return img

      outs = tf.map_fn(preprocess, parent_tensor)
      # train/valid differences
      out_tensor = training * outs + (1 - training) * parent_tensor
    if set_tensors:
      self.out_tensor = out_tensor
    return out_tensor
Esempio n. 17
0
def transpose(images):
    """Transpose an image/images by swapping the first and second dimension.
    (A mirror to tf.image transpose_image)

    Args:
        images: 4-D Tensor of shape `[batch, height, width, channels]` or
            3-D Tensor of shape `[height, width, channels]`.

    Returns:
        If `image` was 4-D, a 4-D float Tensor of shape
        `[batch, target_height, target_width, channels]`
        If `image` was 3-D, a 3-D float Tensor of shape
        `[target_height, target_width, channels]

    Raises:
        ValueError: if the shape of `image` not supported.
    """
    images_shape = get_shape(images)
    if len(images_shape) > 4:
        ValueError("'image' must have either 3 or 4 dimensions, "
                   "received `{}`.".format(images_shape))

    if len(images_shape) == 4:
        return tf.map_fn(lambda img: tf.image.transpose_image(img), images)

    return tf.image.transpose_image(images)
Esempio n. 18
0
def rotate90(images, k=1, is_random=False, seed=None, name=None):
    """Rotate (randomly) images counter-clockwise by 90 degrees.
    (A mirror to tf.image rot90)

    Args:
        images: 4-D Tensor of shape `[batch, height, width, channels]` or
            3-D Tensor of shape `[height, width, channels]`.
        k: A scalar integer. The number of times the image is rotated by 90 degrees.
        is_random: `bool`, If True, adjust randomly.
        seed: A Python integer. Used to create a random seed. See @{tf.set_random_seed}.
        name: A name for this operation (optional).

    Returns:
        If `image` was 4-D, a 4-D float Tensor of shape
        `[batch, target_height, target_width, channels]`
        If `image` was 3-D, a 3-D float Tensor of shape
        `[target_height, target_width, channels]

    Raises:
        ValueError: if the shape of `image` not supported.
    """
    if is_random:
        k = random_ops.random_shuffle([0, 1, 2, 3], seed=seed)[0]

    images_shape = get_shape(images)
    if len(images_shape) > 4:
        ValueError("'image' must have either 3 or 4 dimensions, "
                   "received `{}`.".format(images_shape))

    if len(images_shape) == 4:
        return tf.map_fn(lambda img: tf.image.rot90(img, k, name), images)

    return tf.image.rot90(images, k, name)
Esempio n. 19
0
def get_exemplar_images(images, exemplar_size, targets_pos=None):
  """Crop exemplar image from input images"""
  with tf.name_scope('get_exemplar_image'):
    batch_size, x_height, x_width = images.get_shape().as_list()[:3]
    z_height, z_width = exemplar_size

    if targets_pos is None:
      target_pos_single = [[get_center(x_height), get_center(x_width)]]
      targets_pos_ = tf.tile(target_pos_single, [batch_size, 1])
    else:
      targets_pos_ = targets_pos

    # convert to top-left corner based coordinates
    top = tf.to_int32(tf.round(targets_pos_[:, 0] - get_center(z_height)))
    bottom = tf.to_int32(top + z_height)
    left = tf.to_int32(tf.round(targets_pos_[:, 1] - get_center(z_width)))
    right = tf.to_int32(left + z_width)

    def _slice(x):
      f, t, l, b, r = x
      c = f[t:b, l:r]
      return c

    exemplar_img = tf.map_fn(_slice, (images, top, left, bottom, right), dtype=images.dtype)
    exemplar_img.set_shape([batch_size, z_height, z_width, 3])
    return exemplar_img
Esempio n. 20
0
  def build_inputs_and_outputs(self):

    if self.frame_features:

      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      fn = lambda x: self.build_prediction_graph(x)
      video_id_output, top_indices_output, top_predictions_output = (
          tf.map_fn(fn, serialized_examples, 
                    dtype=(tf.string, tf.int32, tf.float32)))

    else:

      serialized_examples = tf.placeholder(tf.string, shape=(None,))

      video_id_output, top_indices_output, top_predictions_output = (
          self.build_prediction_graph(serialized_examples))

    inputs = {"example_bytes": 
              saved_model_utils.build_tensor_info(serialized_examples)}

    outputs = {
        "video_id": saved_model_utils.build_tensor_info(video_id_output),
        "class_indexes": saved_model_utils.build_tensor_info(top_indices_output),
        "predictions": saved_model_utils.build_tensor_info(top_predictions_output)}

    return inputs, outputs
def attentive_matching(input_sentence, att_matrix, weights):
    """
    Parameters
    ----------
    input_sentence: Tensor
        Tensor of shape (batch_size, num_sentence_words, rnn_hidden_dim)

    att_matrix: Tensor
        Tensor of shape (batch_size, num_sentence_words, rnn_hidden_dim)
    """
    def single_instance(inputs):
        # Shapes: (num_sentence_words, rnn_hidden_dim)
        sentence_a_single = inputs[0]
        sentence_b_single_att = inputs[1]

        # Shapes: (num_sentence_words, multiperspective_dims, rnn_hidden_dim)
        expanded_sentence_a_single = multi_perspective_expand_for_2D(
            sentence_a_single, weights)
        expanded_sentence_b_single_att = multi_perspective_expand_for_2D(
            sentence_b_single_att, weights)
        # Shape: (num_sentence_words, multiperspective_dims)
        return cosine_distance(expanded_sentence_a_single,
                               expanded_sentence_b_single_att)

    elems = (input_sentence, att_matrix)
    # Shape: (batch_size, num_sentence_words, multiperspective_dims)
    return tf.map_fn(single_instance, elems, dtype="float")
Esempio n. 22
0
 def lowercase(self, raw_post):
     split_chars = tf.string_split(tf.reshape(raw_post, [-1]), delimiter="").values
     upchar_inds = self.upchars_lut.lookup(split_chars)
     return tf.reduce_join(tf.map_fn(lambda x: tf.cond(x[0] > 25,
                                                       lambda: x[1],
                                                       lambda: self.lchars[x[0]]),
                                     (upchar_inds, split_chars), dtype=tf.string))
Esempio n. 23
0
 def input_images(self):
     num_images = (self._sqlen + self._args.lookback_length) * self._bsize
     images = tf.map_fn(lambda x: tf.image.decode_jpeg(tf.read_file(x)),
                        tf.reshape(self._imfiles, shape=[num_images]),
                        dtype=tf.uint8)
     images.set_shape([None, ds.HEIGHT, ds.WIDTH, ds.CHANNELS])
     return images
Esempio n. 24
0
def batch_logits(indices, acts):
  init_outs = tf.zeros([1, FLAGS.wvs, 1])

  def logits_continue(*parms):
    cur, idxs, _, _, _ = parms
    return tf.less(cur, tf.size(idxs), name='batch_done')

  def logits_batch_body(*parms):
    i, idxs, ptr, css, act = parms
    i_s = tf.reshape(tf.slice(idxs, tf.pack([i]), [1]), [])
    start, size = get_bounds(i_s)
    outs = forward_prop_nodes(start, size, acts, ptr)
    new_css = tf.cond(tf.equal(i, iZERO),
                      lambda: outs,
                      lambda: tf.concat(0, [css, outs]))
    return i + iONE, indices, ptr + size, new_css, acts
  with tf.device('/cpu:0'):
    iZ =  tf.convert_to_tensor(0, dtype=tf.int32)
  zero_activations(acts)
  while_parms = [iZ, indices, iZ, init_outs, acts]
  _, _, _, outs, _ = tf.while_loop(logits_continue, logits_batch_body, while_parms,
                                   parallel_iterations=1, name='batch_logits')
  lumpy_logits = tf.map_fn(activation_to_logits, outs, name='raw_logits')
  logits = tf.squeeze(lumpy_logits, [2], name='logits')
  return logits
Esempio n. 25
0
  def _sample_n(self, n, seed=None):
    n_draws = tf.cast(self.total_count, dtype=tf.int32)
    k = self.event_shape_tensor()[0]

    # broadcast the total_count and logits to same shape
    n_draws = tf.ones_like(
        self.logits[..., 0], dtype=n_draws.dtype) * n_draws
    logits = tf.ones_like(
        n_draws[..., tf.newaxis], dtype=self.logits.dtype) * self.logits

    # flatten the total_count and logits
    flat_logits = tf.reshape(logits, [-1, k])  # [B1B2...Bm, k]
    flat_ndraws = n * tf.reshape(n_draws, [-1])  # [B1B2...Bm]

    # computes each total_count and logits situation by map_fn
    def _sample_single(args):
      logits, n_draw = args[0], args[1]  # [K], []
      x = tf.multinomial(logits[tf.newaxis, ...], n_draw,
                         seed)  # [1, n*n_draw]
      x = tf.reshape(x, shape=[n, -1])  # [n, n_draw]
      x = tf.reduce_sum(tf.one_hot(x, depth=k), axis=-2)  # [n, k]
      return x

    x = tf.map_fn(
        _sample_single, [flat_logits, flat_ndraws],
        dtype=self.dtype)  # [B1B2...Bm, n, k]

    # reshape the results to proper shape
    x = tf.transpose(x, perm=[1, 0, 2])
    final_shape = tf.concat([[n], self.batch_shape_tensor(), [k]], 0)
    x = tf.reshape(x, final_shape)  # [n, B1, B2,..., Bm, k]
    return x
Esempio n. 26
0
def preprocess_for_inception(images):
  """Preprocess images for inception.

  Args:
    images: images minibatch. Shape [batch size, width, height,
      channels]. Values are in [0..255].

  Returns:
    preprocessed_images
  """

  # Images should have 3 channels.
  assert images.shape[3].value == 3

  # tfgan_eval.preprocess_image function takes values in [0, 1], so rescale.
  with tf.control_dependencies([tf.assert_greater_equal(images, 0.0),
                                tf.assert_less_equal(images, 255.0)]):
    images = tf.identity(images)

  preprocessed_images = tf.map_fn(
      fn=tfgan_eval.preprocess_image,
      elems=images,
      back_prop=False
  )

  return preprocessed_images
Esempio n. 27
0
 def testMap_MultiInputSingleOutput(self):
     with self.test_session():
         nums = np.array([1, 2, 3, 4, 5, 6])
         r = tf.map_fn(lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)), dtype=tf.int64)
         self.assertEqual((6,), r.get_shape())
         received = r.eval()
         self.assertAllEqual(nums * nums + (-nums), received)
Esempio n. 28
0
    def _get_bbox_pred(self, proposed_boxes, gt_boxes_per_class):
        """Computes valid bbox_pred from proposals and gt_boxes for each class.

        Args:
            proposed_boxes: Tensor with shape (num_proposals, 5).
            gt_boxes_per_class: Tensor holding the ground truth boxes for each
                class. Has shape (num_classes, num_gt_boxes_per_class, 4).

        Returns:
            A tensor with shape (num_proposals, num_classes * 4), holding the
            correct bbox_preds.
        """

        def bbox_encode(gt_boxes):
            return encode(
                proposed_boxes, gt_boxes
            )
        bbox_pred_tensor = tf.map_fn(
            bbox_encode, gt_boxes_per_class,
            dtype=tf.float32
        )
        # We need to explicitly unstack the tensor so that tf.concat works
        # properly.
        bbox_pred_list = tf.unstack(bbox_pred_tensor)
        return tf.concat(bbox_pred_list, 1)
def get_labels_from_annotation_batch(annotation_batch_tensor, class_labels):

    batch_labels = tf.map_fn(fn=lambda x: get_labels_from_annotation(annotation_tensor=x, class_labels=class_labels),
                             elems=annotation_batch_tensor,
                             dtype=tf.float32)

    return batch_labels
Esempio n. 30
0
def alignment_summary(features, labels, predictions, mode, params):
    image = tf.cast(features, tf.uint8)
    prediction = predictions['prediction']
    label_points = tf.stack([labels[:, ::2], labels[:, 1::2]], axis=2)
    predict_points = tf.stack(
        [prediction[:, ::2], prediction[:, 1::2]], axis=2)
    def draw_points(args):
        image, label_points, predict_points = args
        def draw_points_pyfn(image, points, color, radius=1):
            if image.shape[-1] == 1:
                image = np.squeeze(image, axis=-1)
            image_pil = Image.fromarray(np.uint8(image)).convert('RGB')
            draw = ImageDraw.Draw(image_pil)
            im_width, im_height = image_pil.size
            for point in points:
                x = point[0] * im_width
                y = point[1] * im_height
                draw.ellipse([(x - radius, y - radius),
                              (x + radius, y + radius)],
                             outline=color, fill=color)
            image = np.array(image_pil)
            return image
        image = tf.py_func(
            functools.partial(draw_points_pyfn, color=(0, 255, 0)),
            (image, label_points), tf.uint8)
        image = tf.py_func(
            functools.partial(draw_points_pyfn, color=(255, 0, 0)),
            (image, predict_points), tf.uint8)
        return image
    image = tf.map_fn(
        draw_points, (image, label_points, predict_points),
        dtype=tf.uint8, back_prop=False)
    tf.summary.image('image', image, 10)
    add_trainable_variables_histogram()
def main():
    if a.seed is None:
        a.seed = random.randint(0, 2**31 - 1)

    tf.set_random_seed(a.seed)
    np.random.seed(a.seed)
    random.seed(a.seed)

    if not os.path.exists(a.output_dir):
        os.makedirs(a.output_dir)

    if a.mode == "test" or a.mode == "export":
        if a.checkpoint is None:
            raise Exception("checkpoint required for test mode")

        # load some options from the checkpoint
        options = {"which_direction", "ngf", "ndf", "lab_colorization"}
        with open(os.path.join(a.checkpoint, "options.json")) as f:
            for key, val in json.loads(f.read()).items():
                if key in options:
                    print("loaded", key, "=", val)
                    setattr(a, key, val)
        # disable these features in test mode
        a.scale_size = CROP_SIZE
        a.flip = False

    for k, v in a._get_kwargs():
        print(k, "=", v)

    with open(os.path.join(a.output_dir, "options.json"), "w") as f:
        f.write(json.dumps(vars(a), sort_keys=True, indent=4))

    examples = load_examples()
    print("examples count = %d" % examples.count)

    # inputs and targets are [batch_size, height, width, channels]
    model = create_model(examples.inputs, examples.targets)

    # undo colorization splitting on images that we use for display/output
    if a.lab_colorization:
        if a.which_direction == "AtoB":
            # inputs is brightness, this will be handled fine as a grayscale image
            # need to augment targets and outputs with brightness
            targets = augment(examples.targets, examples.inputs)
            outputs = augment(model.outputs, examples.inputs)
            # inputs can be deprocessed normally and handled as if they are single channel
            # grayscale images
            inputs = deprocess(examples.inputs)
        elif a.which_direction == "BtoA":
            # inputs will be color channels only, get brightness from targets
            inputs = augment(examples.inputs, examples.targets)
            targets = deprocess(examples.targets)
            outputs = deprocess(model.outputs)
        else:
            raise Exception("invalid direction")
    else:
        inputs = deprocess(examples.inputs)
        targets = deprocess(examples.targets)
        outputs = deprocess(model.outputs)

    def convert(image):
        if a.aspect_ratio != 1.0:
            # upscale to correct aspect ratio
            size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
            image = tf.image.resize_images(
                image, size=size, method=tf.image.ResizeMethod.BICUBIC)

        return tf.image.convert_image_dtype(image,
                                            dtype=tf.uint8,
                                            saturate=True)

    # reverse any processing on images so they can be written to disk or displayed to user
    with tf.name_scope("convert_inputs"):
        converted_inputs = convert(inputs)

    with tf.name_scope("convert_targets"):
        converted_targets = convert(targets)

    with tf.name_scope("convert_outputs"):
        converted_outputs = convert(outputs)

    with tf.name_scope("encode_images"):
        display_fetches = {
            "paths":
            examples.paths,
            "inputs":
            tf.map_fn(tf.image.encode_png,
                      converted_inputs,
                      dtype=tf.string,
                      name="input_pngs"),
            "targets":
            tf.map_fn(tf.image.encode_png,
                      converted_targets,
                      dtype=tf.string,
                      name="target_pngs"),
            "outputs":
            tf.map_fn(tf.image.encode_png,
                      converted_outputs,
                      dtype=tf.string,
                      name="output_pngs"),
        }

    # summaries
    with tf.name_scope("inputs_summary"):
        tf.summary.image("inputs", converted_inputs)

    with tf.name_scope("targets_summary"):
        tf.summary.image("targets", converted_targets)

    with tf.name_scope("outputs_summary"):
        tf.summary.image("outputs", converted_outputs)

    with tf.name_scope("predict_real_summary"):
        tf.summary.image(
            "predict_real",
            tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8))

    with tf.name_scope("predict_fake_summary"):
        tf.summary.image(
            "predict_fake",
            tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8))

    tf.summary.scalar("discriminator_loss", model.discrim_loss)
    tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
    tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)

    for var in tf.trainable_variables():
        tf.summary.histogram(var.op.name + "/values", var)

    for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
        tf.summary.histogram(var.op.name + "/gradients", grad)

    with tf.name_scope("parameter_count"):
        parameter_count = tf.reduce_sum(
            [tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])

    saver = tf.train.Saver(max_to_keep=1)

    logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
    sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
    with sv.managed_session() as sess:
        print("parameter_count =", sess.run(parameter_count))

        if a.checkpoint is not None:
            print("#############################")
            print("loading model from checkpoint")
            print("#############################")
            try:
                checkpoint = tf.train.latest_checkpoint(a.checkpoint)
                saver.restore(sess, checkpoint)
            except:
                print("loading was unsuccessful")
                print("#############################")

        max_steps = 2**32
        if a.max_epochs is not None:
            max_steps = examples.steps_per_epoch * a.max_epochs
        if a.max_steps is not None:
            max_steps = a.max_steps

        if a.mode == "test":
            # testing
            # at most, process the test data once
            start = time.time()
            max_steps = min(examples.steps_per_epoch, max_steps)
            for step in range(max_steps):
                results = sess.run(display_fetches)
                filesets = save_images(results)
                for i, f in enumerate(filesets):
                    print("evaluated image", f["name"])
                index_path = append_index(filesets)
            print("wrote index at", index_path)
            print("rate", (time.time() - start) / max_steps)
        else:
            # training
            start = time.time()

            for step in range(max_steps):

                def should(freq):
                    return freq > 0 and ((step + 1) % freq == 0
                                         or step == max_steps - 1)

                options = None
                run_metadata = None
                if should(a.trace_freq):
                    options = tf.RunOptions(
                        trace_level=tf.RunOptions.FULL_TRACE)
                    run_metadata = tf.RunMetadata()

                fetches = {
                    "train": model.train,
                    "global_step": sv.global_step,
                }

                if should(a.progress_freq):
                    fetches["discrim_loss"] = model.discrim_loss
                    fetches["gen_loss_GAN"] = model.gen_loss_GAN
                    fetches["gen_loss_L1"] = model.gen_loss_L1

                if should(a.summary_freq):
                    fetches["summary"] = sv.summary_op

                if should(a.display_freq):
                    fetches["display"] = display_fetches

                results = sess.run(fetches,
                                   options=options,
                                   run_metadata=run_metadata)

                if should(a.summary_freq):
                    print("recording summary")
                    sv.summary_writer.add_summary(results["summary"],
                                                  results["global_step"])

                if should(a.display_freq):
                    print("saving display images")
                    filesets = save_images(results["display"],
                                           step=results["global_step"])
                    append_index(filesets, step=True)

                if should(a.trace_freq):
                    print("recording trace")
                    sv.summary_writer.add_run_metadata(
                        run_metadata, "step_%d" % results["global_step"])

                if should(a.progress_freq):
                    # global_step will have the correct step count if we resume from a checkpoint
                    train_epoch = math.ceil(results["global_step"] /
                                            examples.steps_per_epoch)
                    train_step = (results["global_step"] -
                                  1) % examples.steps_per_epoch + 1
                    rate = (step + 1) * a.batch_size / (time.time() - start)
                    remaining = (max_steps - step) * a.batch_size / rate
                    print(
                        "progress  epoch %d  step %d  image/sec %0.1f  remaining %dm"
                        % (train_epoch, train_step, rate, remaining / 60))
                    print("discrim_loss", results["discrim_loss"])
                    print("gen_loss_GAN", results["gen_loss_GAN"])
                    print("gen_loss_L1", results["gen_loss_L1"])

                    if results["gen_loss_L1"] < float(a.desired_l1_loss):
                        print("###################")
                        print("Reached desired error")
                        print(
                            "progress  epoch %d  step %d  image/sec %0.1f  remaining %dm"
                            % (train_epoch, train_step, rate, remaining / 60))
                        print("saving model")
                        saver.save(sess,
                                   os.path.join(a.output_dir, "model"),
                                   global_step=sv.global_step)
                        break

                if should(a.save_freq):
                    print("saving model")
                    saver.save(sess,
                               os.path.join(a.output_dir, "model"),
                               global_step=sv.global_step)

                if sv.should_stop():
                    break
Esempio n. 32
0
def esmm_model_fn(features, labels, mode, params):
  batch_weight = tf.feature_column.input_layer(features, params['weight_columns'])
  inputs, shared_weights = build_input(features, params)
  hidden_units = params['hidden_units']
  linear_parent_scope = 'linear'
  dnn_parent_scope = 'dnn'
  is_dynamic = params['dynamic']
  print("is_dynamic:", is_dynamic)
  reg = 1e-4
  if params['model'] == 'linear':
    with tf.variable_scope(linear_parent_scope, values=tuple(six.itervalues(features)), reuse=tf.AUTO_REUSE):
      with tf.variable_scope('linear_ctr'):
        ctr_logit_fn = linear._linear_logit_fn_builder(1, params['linear_columns'])
        ctr_logits = ctr_logit_fn(features=features)
      with tf.variable_scope('linear_cvr'):
        cvr_logit_fn = linear._linear_logit_fn_builder(1, params['linear_columns'])
        cvr_logits = cvr_logit_fn(features=features)
  if params['model'] == 'dnn':
    with tf.variable_scope(dnn_parent_scope):
      with tf.variable_scope('dnn_ctr'):
        ctr_logits = build_deep_layers(inputs, hidden_units, mode, params['ctr_reg'])
        #ctr_logit_fn = dnn._dnn_logit_fn_builder(1, hidden_units, params['dnn_columns'], tf.nn.relu, None, None, True)
        #ctr_logits = ctr_logit_fn(features=features, mode=mode)
      with tf.variable_scope('dnn_cvr'):
        cvr_logits = build_deep_layers(inputs, hidden_units, mode, params['cvr_reg'])
        #cvr_logit_fn = dnn._dnn_logit_fn_builder(1, hidden_units, params['dnn_columns'], tf.nn.relu, None, None, True)
        #cvr_logits = cvr_logit_fn(features=features, mode=mode)
  logits = {'ctr': ctr_logits, 'ctcvr': ctr_logits*cvr_logits}
  ctr_preds = tf.nn.sigmoid(ctr_logits)
  cvr_preds = tf.nn.sigmoid(cvr_logits)
  #ctcvr_preds = tf.stop_gradient(ctr_preds) * cvr_preds
  ctcvr_preds = ctr_preds * cvr_preds
  tf.summary.histogram("esmm/ctr_preds", ctr_preds) 
  tf.summary.histogram("esmm/ctcvr_preds", ctcvr_preds)
  if mode == tf.estimator.ModeKeys.PREDICT:
    #redundant_items = ctr_preds
    predictions = {
      'prob': tf.concat([ctcvr_preds, ctr_preds], 1)
    }
    export_outputs = {
      tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(predictions)  #线上预测需要的
    }
    return tf.estimator.EstimatorSpec(mode, predictions=predictions, export_outputs=export_outputs)

  else:
    #shared_weights = tf.trainable_variables(dnn_parent_scope + '/SharedLayer/kernel')[0]
    ctr_labels = labels['ctr']
    ctcvr_labels = labels['ctcvr']
    linear_optimizer = tf.train.FtrlOptimizer(0.01, l1_regularization_strength=0.001, l2_regularization_strength=0.001)
    dnn_optimizer = optimizers.get_optimizer_instance('Adam', params['learning_rate'])
    loss_optimizer = optimizers.get_optimizer_instance('Adam', 0.001)
    ctr_loss = tf.losses.log_loss(ctr_labels, ctr_preds, reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE, weights=batch_weight)
    ctcvr_loss = tf.losses.log_loss(ctcvr_labels, ctcvr_preds, reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE)
    #reg_loss = tf.reduce_sum(ops.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
    ctr_auc = tf.metrics.auc(labels=ctr_labels, predictions=ctr_preds, weights=batch_weight)
    ctcvr_auc = tf.metrics.auc(labels=ctcvr_labels, predictions=ctcvr_preds)
    mask = tf.map_fn(lambda x:tf.cond(tf.equal(x, 1), lambda: True, lambda: False), tf.squeeze(labels['ctr']), dtype=tf.bool)
    cvr_preds = tf.boolean_mask(cvr_preds, mask)
    cvr_labels = tf.boolean_mask(labels['ctcvr'], mask)
    cvr_auc = tf.metrics.auc(labels=cvr_labels, predictions=cvr_preds)
    cvr_loss = tf.losses.log_loss(cvr_labels, cvr_preds, reduction=tf.losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)
    tf.summary.scalar("cvr_auc", cvr_auc[1])
    tf.summary.scalar("cvr_loss", cvr_loss)
    tf.summary.scalar('ctr_loss', ctr_loss)
    tf.summary.scalar('ctcvr_loss', ctcvr_loss)
    tf.summary.scalar('ctr_auc', ctr_auc[1])
    tf.summary.scalar('ctcvr_auc', ctcvr_auc[1])
    loss = tf.add_n([ctr_loss, ctcvr_loss])
    #weight_loss, update_list, w_list, loss_gradnorm = get_weight_loss([ctr_loss, ctcvr_loss], is_dynamic, shared_weights)
    #print("get_weight_loss:", weight_loss, update_list)
    def _train_op_fn(loss):
      train_ops = []
      global_step = tf.train.get_global_step()
      if params['model'] in ('dnn'):
        fm_var_list = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope='fm') 
        dnn_var_list = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope=dnn_parent_scope) + ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope='dnn_embed')
        train_ops.append(
          dnn_optimizer.minimize(
            loss,
            var_list=dnn_var_list))
        train_ops.append(
          linear_optimizer.minimize(
            loss,
            var_list=fm_var_list))
      if params['model'] in ('linear'):
        train_ops.append(
          linear_optimizer.minimize(
              loss,
              var_list=tf.get_collection(
                  tf.GraphKeys.TRAINABLE_VARIABLES,
                  scope=linear_parent_scope)))
      '''
      if w_list is not None and update_list is not None and loss_gradnorm is not None:
        train_ops.append(
            loss_optimizer.minimize(
                loss_gradnorm,
                var_list=w_list))
        train_ops.append(update_list)
      '''
      train_op = control_flow_ops.group(*train_ops)
      with ops.control_dependencies([train_op]):
        return state_ops.assign_add(global_step, 1).op
    hooks = tf.train.LoggingTensorHook({'ctr_loss':ctr_loss, 'ctcvr_loss':ctcvr_loss, 'cvr_loss':cvr_loss}, every_n_iter=10000)
    train_op = _train_op_fn(loss)
    train_op = head_v1._append_update_ops(train_op)
    metrics = {'ctr_auc': ctr_auc, 'ctcvr_auc': ctcvr_auc, 'cvr_auc': cvr_auc}
    #return _TPUEstimatorSpec(mode, loss=loss, train_op=train_op).as_estimator_spec()
    return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op, eval_metric_ops=metrics)
    def build_graph(self, data_paths, batch_size, graph_mod):
        """Builds generic graph for training or eval."""
        tensors = GraphReferences()
        is_training = graph_mod == GraphMod.TRAIN
        if data_paths:
            tensors.keys, tensors.examples = util.read_examples(
                data_paths,
                batch_size,
                shuffle=is_training,
                num_epochs=None if is_training else 2)
        else:
            logging.info("No data path")
            tensors.examples = tf.placeholder(tf.string,
                                              name='input',
                                              shape=(None, ))

        if graph_mod == GraphMod.PREDICT:
            pass
        else:
            # For training and evaluation we assume data is preprocessed, so the
            # inputs are tf-examples.
            # Generate placeholders for examples.
            with tf.name_scope('inputs'):
                feature_map = {
                    'image_uri':
                    tf.FixedLenFeature(shape=[],
                                       dtype=tf.string,
                                       default_value=['']),
                    'image_bytes':
                    tf.FixedLenFeature(shape=[],
                                       dtype=tf.string,
                                       default_value=['']),
                    'label':
                    tf.FixedLenFeature(shape=[1],
                                       dtype=tf.int64,
                                       default_value=[self.label_count])
                }
                #tensors.examples = tf.Print(tensors.examples, [tf.shape(tensors.examples)], message="Parsing examples: ")
                parsed = tf.parse_example(tensors.examples,
                                          features=feature_map)
                labels = tf.squeeze(parsed['label'])
                uris = tf.squeeze(parsed['image_uri'])
                images_str_tensor = parsed['image_bytes']

            def decode_and_resize(image_str_tensor):
                """Decodes jpeg string, resizes it and returns a uint8 tensor."""
                image = tf.image.decode_jpeg(image_str_tensor, channels=1)

                # Note resize expects a batch_size, but tf_map supresses that index,
                # thus we have to expand then squeeze.  Resize returns float32 in the
                # range [0, uint8_max]
                """
          image = tf.expand_dims(image, 0)
          image = tf.image.resize_bilinear(
              image, [height, width], align_corners=False)*/
          image = tf.squeeze(image, squeeze_dims=[0])
          """

                image = tf.cast(image, dtype=tf.uint8)

                # convert_image_dtype, also scales [0, uint8_max] -> [0 ,1).
                return tf.image.convert_image_dtype(image, dtype=tf.float32)

        #images_str_tensor = tf.Print(images_str_tensor, [tf.shape(images_str_tensor)], message="Decoding images: ")
        images = tf.map_fn(decode_and_resize,
                           images_str_tensor,
                           back_prop=False,
                           dtype=tf.float32)

        # We assume a default label, so the total number of labels is equal to
        # label_count+1.
        all_labels_count = self.label_count + 1

        with tf.name_scope('model'):

            fc_padding = 'VALID'
            with tf.variable_scope('model', 'vgg_16', [images]) as sc:
                end_points_collection = sc.original_name_scope + '_end_points'
                # Collect outputs for conv2d, fully_connected and max_pool2d.
                with slim.arg_scope(
                    [slim.conv2d, slim.fully_connected, slim.max_pool2d],
                        outputs_collections=end_points_collection):
                    # images = tf.Print(images, [tf.shape(images)], message="Shape of input: ", summarize=4)
                    net = slim.repeat(images,
                                      2,
                                      slim.conv2d,
                                      64, [3, 3],
                                      scope='conv1')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv1')
                    net = slim.max_pool2d(net, [2, 2], scope='pool1')
                    net = slim.repeat(net,
                                      2,
                                      slim.conv2d,
                                      128, [3, 3],
                                      scope='conv2')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv2')
                    net = slim.max_pool2d(net, [2, 2], scope='pool2')
                    net = slim.repeat(net,
                                      3,
                                      slim.conv2d,
                                      256, [3, 3],
                                      scope='conv3')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv3')
                    net = slim.max_pool2d(net, [2, 2], scope='pool3')
                    net = slim.repeat(net,
                                      3,
                                      slim.conv2d,
                                      512, [3, 3],
                                      scope='conv4')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv4')
                    net = slim.max_pool2d(net, [2, 2], scope='pool4')
                    net = slim.repeat(net,
                                      3,
                                      slim.conv2d,
                                      512, [3, 3],
                                      scope='conv5')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='conv5')
                    net = slim.max_pool2d(net, [2, 2], scope='pool5')

                    # Use conv2d instead of fully_connected layers.
                    net = slim.conv2d(net,
                                      4096, [7, 7],
                                      padding=fc_padding,
                                      scope='fc6')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='fc6')
                    net = slim.dropout(net,
                                       0.5,
                                       is_training=True,
                                       scope='dropout6')
                    net = slim.conv2d(net, 4096, [1, 1], scope='fc7')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='fc7')
                    # Convert end_points_collection into a end_point dict.
                    end_points = slim.utils.convert_collection_to_dict(
                        end_points_collection)

                    net = slim.dropout(net,
                                       0.5,
                                       is_training=True,
                                       scope='dropout7')
                    net = slim.conv2d(net,
                                      all_labels_count, [1, 1],
                                      activation_fn=None,
                                      normalizer_fn=None,
                                      scope='fc8')
                    #net = tf.Print(net, [tf.shape(net)], summarize=4, message='fc8')

                    net = tf.squeeze(net, [1, 2], name='fc8/squeezed')
                    end_points[sc.name + '/fc8'] = net
        logits = net
        softmax = tf.nn.softmax(logits)
        #softmax = tf.Print(softmax, [tf.shape(softmax)], summarize=4, message='softmax')
        # Prediction is the index of the label with the highest score. We are
        # interested only in the top score.
        prediction = tf.argmax(softmax, 1)
        tensors.predictions = [prediction, softmax, images]

        if graph_mod == GraphMod.PREDICT:
            return tensors

        with tf.name_scope('evaluate'):
            loss_value = loss(logits, labels)

        # Add to the Graph the Ops that calculate and apply gradients.
        if is_training:
            tensors.train, tensors.global_step = training(loss_value)
        else:
            tensors.global_step = tf.Variable(0,
                                              name='global_step',
                                              trainable=False)

        # Add means across all batches.
        loss_updates, loss_op = util.loss(loss_value)
        accuracy_updates, accuracy_op = util.accuracy(logits, labels)

        if not is_training:
            #accuracy_op = tf.Print(accuracy_op, [accuracy_op], message="Accuracy")
            #loss_op = tf.Print(loss_op, [loss_op], message="Loss")
            tf.summary.scalar('accuracy', accuracy_op)
            tf.summary.scalar('loss', loss_op)

        tensors.metric_updates = loss_updates + accuracy_updates
        tensors.metric_values = [loss_op, accuracy_op]
        return tensors
 def augment_image(image):
     image = tf.image.resize_images(image, [self._augment_size, self._augment_size])
     image = tf.random_crop(image, [self._batch_size] + self._image_shape, seed=seed)
     image = tf.map_fn(lambda x: tf.image.random_flip_left_right(x, seed), image)
     return image
Esempio n. 35
0
    def build_template(self):
        model_config = self.model_config
        track_config = self.track_config
        size_z = model_config['z_image_size']
        ratio = self.target_size[0] / self.target_size[1]

        # Exemplar image lies at the center of the search image in the first frame
        center_scale = int(get_center(track_config['num_scales']))
        search_images = tf.expand_dims(self.search_images[center_scale], 0)
        exemplar_images = get_exemplar_images(
            search_images, [size_z, size_z],
            np.array([[
                get_center(track_config['x_image_size']),
                get_center(track_config['x_image_size'])
            ]]))

        def boundary_suppression(embeds, embeds2, ratio):
            offsets = tf.cond(
                tf.greater(ratio, 1.5), lambda: [0, 4, 0, 4],
                lambda: tf.cond(tf.less(ratio, 0.67), lambda: [4, 0, 4, 0],
                                lambda: [2, 2, 2, 2]))
            embeds = tf.image.resize_image_with_crop_or_pad(
                embeds, t_shape[1] - offsets[0], t_shape[2] - offsets[1])
            embeds = tf.image.resize_image_with_crop_or_pad(
                embeds, t_shape[1], t_shape[2])
            embeds2 = tf.image.resize_image_with_crop_or_pad(
                embeds2, t_shape2[1] - offsets[2], t_shape2[2] - offsets[3])
            embeds2 = tf.image.resize_image_with_crop_or_pad(
                embeds2, t_shape2[1], t_shape2[2])
            return embeds, embeds2

        def background_suppression(embeds, ratio):
            offsets = tf.cond(
                tf.greater(ratio, 1.5),  # 1.2 / 0.83; 1.5 / 0.67
                lambda: [1., 1.2 / ratio],
                lambda: tf.cond(
                    tf.less(ratio, 0.67), lambda: [1.2 * ratio, 1.], lambda: tf
                    .cond(
                        tf.greater(ratio, 1.2), lambda: [1., 1.1 / ratio],
                        lambda: tf.cond(tf.less(ratio, 0.83), lambda: [
                            1.1 * ratio, 1.
                        ], lambda: [0.7, 0.7]))))

            h = tf.cast(size_z * offsets[0], tf.int32)
            w = tf.cast(size_z * offsets[1], tf.int32)

            embeds_mean = tf.reduce_mean(embeds, axis=(0, 1), keepdims=True)
            embeds = embeds - embeds_mean
            embeds = tf.image.resize_image_with_crop_or_pad(embeds, h, w)
            embeds = tf.image.resize_image_with_crop_or_pad(
                embeds, size_z, size_z)
            return embeds + embeds_mean

        exemplar_images = tf.map_fn(
            lambda x: background_suppression(x[0], x[1]),
            (exemplar_images, tf.expand_dims(ratio, 0)),
            dtype=exemplar_images.dtype)
        self.exemplar_images = exemplar_images

        templates, templates2 = self.get_image_embedding(exemplar_images)
        t_shape = templates.get_shape().as_list()
        t_shape2 = templates2.get_shape().as_list()

        templates, templates2 = tf.map_fn(
            lambda x: boundary_suppression(x[0], x[1], x[2]),
            (templates, templates2, tf.expand_dims(ratio, 0)),
            dtype=(templates.dtype, templates2.dtype))

        templates = templates
        templates2 = templates2

        with tf.variable_scope('target_template'):
            # Store template in Variable such that we don't have to feed this template every time.
            with tf.variable_scope('State'):
                state = tf.get_variable('exemplar',
                                        initializer=tf.zeros(
                                            templates.get_shape().as_list(),
                                            dtype=templates.dtype),
                                        trainable=False)
                state2 = tf.get_variable('exemplar2',
                                         initializer=tf.zeros(
                                             templates2.get_shape().as_list(),
                                             dtype=templates2.dtype),
                                         trainable=False)
                with tf.control_dependencies([templates, templates2]):
                    self.init = tf.assign(state,
                                          templates,
                                          validate_shape=True)
                    self.init2 = tf.assign(state2,
                                           templates2,
                                           validate_shape=True)

                self.templates = state
                self.templates2 = state2

                # Store Pseudo Templates
                def _euc_distance(x, z):
                    z = tf.expand_dims(z, 0)
                    return tf.reduce_sum(tf.abs(x - z), -1)

                num_k = 3  # 3
                state_pseu = []
                state_pseu2 = []
                image_pseu = []
                self.init_pseu = []
                self.init2_pseu = []
                self.init_pseu_img = []
                for i in range(num_k):
                    state_pseu.append(
                        tf.get_variable('exemplar_pseu' + str(i),
                                        initializer=tf.zeros(
                                            templates.get_shape().as_list(),
                                            dtype=templates.dtype),
                                        trainable=False))
                    state_pseu2.append(
                        tf.get_variable('exemplar2_pseu' + str(i),
                                        initializer=tf.zeros(
                                            templates2.get_shape().as_list(),
                                            dtype=templates2.dtype),
                                        trainable=False))
                    image_pseu.append(
                        tf.get_variable(
                            'exemplar_pseu_image' + str(i),
                            initializer=tf.zeros(
                                exemplar_images.get_shape().as_list(),
                                dtype=exemplar_images.dtype),
                            trainable=False))
                    with tf.control_dependencies(
                        [templates, templates2, exemplar_images]):
                        self.init_pseu.append(
                            tf.assign(state_pseu[i],
                                      templates,
                                      validate_shape=True))
                        self.init2_pseu.append(
                            tf.assign(state_pseu2[i],
                                      templates2,
                                      validate_shape=True))
                        self.init_pseu_img.append(
                            tf.assign(image_pseu[i],
                                      exemplar_images,
                                      validate_shape=True))

                self.image_pseu = image_pseu
                self.pseu_temp = state_pseu
                self.pseu_temp2 = state_pseu2

                state_pseus = tf.concat([self.templates] + state_pseu +
                                        [templates], 0)
                sp_shape = state_pseus.get_shape().as_list()[0]
                state_pseus_c = tf.reshape(state_pseus, [sp_shape, -1])
                state_pseus_dis = tf.map_fn(
                    lambda x: _euc_distance(state_pseus_c, x),
                    state_pseus_c,
                    dtype=state_pseus_c.dtype)
                state_pseus_dis = tf.reshape(state_pseus_dis,
                                             [sp_shape, sp_shape])[1:, :]
                state_pseus_dis = tf.reduce_sum(state_pseus_dis, -1)
                self.state_pseus_dis = state_pseus_dis
                _, state_pseus_idx = tf.nn.top_k(state_pseus_dis,
                                                 k=len(state_pseu))

                image_pseu_extra = tf.concat(image_pseu + [exemplar_images], 0)
                state_pseus2 = tf.concat(state_pseu2 + [templates2], 0)
                self.up_img = []
                self.up_pseu = []
                self.up2_pseu = []
                for i in range(len(state_pseu)):
                    with tf.control_dependencies([
                            state_pseus_idx, image_pseu_extra, state_pseus,
                            state_pseus2
                    ]):
                        self.up_pseu.append(
                            tf.assign(state_pseu[i],
                                      tf.expand_dims(
                                          state_pseus[state_pseus_idx[i] + 1],
                                          0),
                                      validate_shape=True))
                        self.up2_pseu.append(
                            tf.assign(state_pseu2[i],
                                      tf.expand_dims(
                                          state_pseus2[state_pseus_idx[i]], 0),
                                      validate_shape=True))
                        self.up_img.append(
                            tf.assign(image_pseu[i],
                                      tf.expand_dims(
                                          image_pseu_extra[state_pseus_idx[i]],
                                          0),
                                      validate_shape=True))
    def call(self, inputs):
        if self.r_num == 1:
            # if there is no routing (and this is so when r_num is 1 and all c are equal)
            # then this is a common convolution
            outputs = K.conv2d(
                K.reshape(inputs,
                          (-1, self.h_i, self.w_i, self.ch_i * self.n_i)),
                K.reshape(
                    self.w, self.kernel_size +
                    (self.ch_i * self.n_i, self.ch_j * self.n_j)),
                data_format='channels_last',
                strides=self.strides,
                padding=self.padding,
                dilation_rate=self.dilation_rate)

            outputs = squeeze(
                K.reshape(outputs,
                          ((-1, self.h_j, self.w_j, self.ch_j, self.n_j))))

        else:
            bt = K.shape(inputs)[0]
            ksz = self.kernel_size[0] * self.kernel_size[1]

            xr = K.reshape(inputs,
                           (-1, self.h_i, self.w_i, self.ch_i * self.n_i))

            pt = tf.extract_image_patches(xr, (1, ) + self.kernel_size + (1, ),
                                          (1, ) + self.strides + (1, ), (
                                              1,
                                              1,
                                              1,
                                              1,
                                          ), 'VALID')

            pt = K.reshape(pt, (-1, ksz * self.ch_i, self.n_i))

            wr = K.reshape(self.w,
                           (ksz * self.ch_i, self.n_i, self.ch_j * self.n_j))

            global useGPU

            # it sometimes works faster on GPU when batch is devided into two parts
            # bp = K.expand_dims(bt // 2, axis=0) if useGPU else K.constant([2], dtype=tf.int32)
            bp = K.expand_dims(bt // 1, axis=0) if useGPU else K.constant(
                [2], dtype=tf.int32)

            if self.strides != (1, 1):
                zr_shape = K.concatenate([
                    bp,
                    K.constant([self.w_j, ksz * self.ch_i * self.ch_j],
                               dtype=tf.int32)
                ])
                zr = tf.zeros(shape=zr_shape)
                zc_shape = K.concatenate([
                    bp,
                    K.constant([self.ah_j, ksz * self.ch_i * self.ch_j],
                               dtype=tf.int32)
                ])
                zc = tf.zeros(shape=zc_shape)

            def rt(ptb):
                ptb = K.reshape(ptb, (-1, ksz * self.ch_i, self.n_i))

                if useGPU:
                    ub = tf.einsum('bin,inj->bij', ptb, wr)
                else:
                    ul = []
                    for i in range(ksz * self.ch_i):
                        ul.append(K.dot(ptb[:, i], wr[i]))
                    ub = K.stack(ul, axis=1)

                #b = tf.constant_initializer(0.)((bp, self.h_i*self.w_i*self.ch_i,
                #                           ksz * self.ch_j))
                b = 0.0

                j_all = self.h_j * self.w_j * self.ch_j
                j_add = j_all - ksz * self.ch_j

                for r in range(self.r_num):
                    ex = K.exp(b * self.b_alphas[r])
                    if r > 0:
                        c = ex / (
                            (K.sum(ex, axis=-1, keepdims=True) + K.epsilon()) +
                            j_add) * j_all
                        c = K.reshape(c, (-1, self.h_i, self.w_i,
                                          self.ch_i * ksz * self.ch_j))
                        c = K.stop_gradient(c)

                        pc = tf.extract_image_patches(
                            c, (1, ) + self.kernel_size + (1, ),
                            (1, ) + self.strides + (1, ), (
                                1,
                                1,
                                1,
                                1,
                            ), 'VALID')
                        pc = K.reshape(pc, (-1, self.h_j, self.w_j, ksz,
                                            self.ch_i, self.kernel_size[0] *
                                            self.kernel_size[1], self.ch_j))
                        pcl = []
                        for n in range(ksz):
                            pcl.append(
                                pc[:, :, :, n, :,
                                   self.kernel_size[0] * self.kernel_size[1] -
                                   1 - n])
                        pcc = K.stack(pcl, axis=3)

                        if useGPU:
                            pcc = K.reshape(pcc,
                                            (-1, self.h_j * self.w_j * ksz *
                                             self.ch_i * self.ch_j, 1))
                            ub = K.reshape(ub,
                                           (-1, self.h_j * self.w_j * ksz *
                                            self.ch_i * self.ch_j, self.n_j))
                            cu = pcc * ub
                        else:
                            pcc = K.reshape(pcc, (-1, 1))
                            ub = K.reshape(ub, (-1, self.n_j, 1))
                            cul = []
                            for n in range(self.n_j):
                                cul.append(ub[:, n] * pcc)
                            cu = K.stack(cul, axis=-2)

                    else:
                        cu = ub

                    cu = K.reshape(cu, (-1, self.h_j * self.w_j,
                                        ksz * self.ch_i, self.ch_j, self.n_j))

                    s = K.sum(cu, axis=-3)

                    v = squeeze(s)
                    if r == self.r_num - 1:
                        break

                    v = K.stop_gradient(v)

                    ubr = K.reshape(K.stop_gradient(ub),
                                    (-1, self.h_j * self.w_j, ksz * self.ch_i,
                                     self.ch_j, self.n_j))

                    if True:
                        #if useGPU:
                        a = tf.einsum('bjck,bjick->bjic', v, ubr)
                    else:
                        al = []
                        for i in range(ksz * self.ch_i):
                            al.append(
                                K.batch_dot(
                                    K.reshape(ubr[:, :, i],
                                              (-1, self.h_j * self.w_j *
                                               self.ch_j, 1, self.n_j)),
                                    K.reshape(v, (-1, self.h_j * self.w_j *
                                                  self.ch_j, self.n_j, 1))))
                        a = K.stack(al, axis=1)
                        a = K.reshape(a, (-1, ksz * self.ch_i,
                                          self.h_j * self.w_j, self.ch_j))
                        a = K.permute_dimensions(a, [0, 2, 1, 3])

                    ph, pw = 2, 2
                    a = K.reshape(
                        a,
                        (-1, self.h_j, self.w_j, ksz * self.ch_i * self.ch_j))

                    if self.strides == (1, 1):
                        aa = a
                    else:
                        rl = []

                        for r in range(self.ah_j):
                            rl.append(zr if r % ph else a[:, r // ph])
                        rs = K.stack(rl, axis=1)
                        cl = []
                        for c in range(self.aw_j):
                            cl.append(zc if c % pw else rs[:, :, c // pw])
                        aa = K.stack(cl, axis=-2)

                    aa = K.spatial_2d_padding(aa, ((ph, ph), (pw, pw)),
                                              data_format='channels_last')
                    pa = tf.extract_image_patches(
                        aa,
                        (1, ) + self.kernel_size + (1, ),
                        (
                            1,
                            1,
                            1,
                            1,
                        ),  #(1,)+strides+(1,),
                        (
                            1,
                            1,
                            1,
                            1,
                        ),
                        'VALID')
                    pa = K.reshape(pa, (-1, self.h_i * self.w_i, ksz, ksz,
                                        self.ch_i, self.ch_j))

                    pal = []
                    for n in range(ksz):
                        pal.append(pa[:, :, n, ksz - 1 - n])
                    paa = K.stack(pal, axis=3)

                    paa = K.reshape(
                        paa,
                        (-1, self.h_i * self.w_i * self.ch_i, ksz * self.ch_j))
                    b = b + paa

                return v

            v = tf.map_fn(rt,
                          K.reshape(pt, (-1, bp[0], self.h_j, self.w_j,
                                         ksz * self.ch_i, self.n_i)),
                          parallel_iterations=100,
                          back_prop=True,
                          infer_shape=False)

            outputs = v
            outputs = K.reshape(outputs,
                                (-1, self.h_j, self.w_j, self.ch_j, self.n_j))
        return outputs
Esempio n. 37
0
def parse_data(proto,
               include_flow,
               height=None,
               width=None,
               include_occlusion=False,
               include_invalid=False,
               resize_gt_flow=True,
               gt_flow_shape=None):
    """Parse a data proto with flow.

  Args:
    proto: path to data proto file
    include_flow: bool, whether or not to include flow in the output
    height: int or None height to resize image to
    width: int or None width to resize image to
    include_occlusion: bool, whether or not to also return occluded pixels (will
      throw error if occluded pixels are not present)
    include_invalid: bool, whether or not to also return invalid pixels (will
      throw error if invalid pixels are not present)
    resize_gt_flow: bool, wether or not to resize flow ground truth as the image
    gt_flow_shape: list, shape of the original ground truth flow (only required
      to set a fixed ground truth flow shape for tensorflow estimator in case of
      supervised training at full resolution resize_gt_flow=False)

  Returns:
    images, flow: A tuple of (image1, image2), flow
  """

    # Parse context and image sequence from protobuffer.
    context_features = {
        'height': tf.io.FixedLenFeature([], tf.int64),
        'width': tf.io.FixedLenFeature([], tf.int64),
    }
    sequence_features = {
        'images': tf.io.FixedLenSequenceFeature([], tf.string),
    }

    if include_invalid:
        sequence_features['invalid_masks'] = tf.io.FixedLenSequenceFeature(
            [], tf.string)

    if include_flow:
        pass  # context_features['flow_uv'] = tf.io.FixedLenFeature([], tf.string)

    if include_occlusion:
        context_features['occlusion_mask'] = tf.io.FixedLenFeature([],
                                                                   tf.string)

    context_parsed, sequence_parsed = tf.io.parse_single_sequence_example(
        proto,
        context_features=context_features,
        sequence_features=sequence_features,
    )

    def deserialize(s, dtype, dims):
        return tf.reshape(
            tf.io.decode_raw(s, dtype),
            [context_parsed['height'], context_parsed['width'], dims])

    images = tf.map_fn(lambda s: deserialize(s, tf.uint8, 3),
                       sequence_parsed['images'],
                       dtype=tf.uint8)

    images = tf.image.convert_image_dtype(images, tf.float32)
    if height is not None and width is not None:
        images = uflow_utils.resize(images, height, width, is_flow=False)
    output = [images]

    if include_flow:
        '''
    flow_uv = deserialize(context_parsed['flow_uv'], tf.float32, 2)
    flow_uv = flow_uv[Ellipsis, ::-1]
    if height is not None and width is not None and resize_gt_flow:
      flow_uv = uflow_utils.resize(flow_uv, height, width, is_flow=True)
    else:
      if gt_flow_shape is not None:
        flow_uv.set_shape(gt_flow_shape)
    # To be consistent with uflow internals, we flip the ordering of flow.
    
    output.append(flow_uv)
    # create valid mask
    flow_valid = tf.ones_like(flow_uv[Ellipsis, :1], dtype=tf.float32)
    output.append(flow_valid)
    '''
        output.append([])
        output.append([])

    if include_occlusion:
        occlusion_mask = deserialize(context_parsed['occlusion_mask'],
                                     tf.uint8, 1)
        if height is not None and width is not None:
            occlusion_mask = uflow_utils.resize(occlusion_mask,
                                                height,
                                                width,
                                                is_flow=False)
        output.append(occlusion_mask)

    if include_invalid:
        invalid_masks = tf.map_fn(lambda s: deserialize(s, tf.uint8, 1),
                                  sequence_parsed['invalid_masks'],
                                  dtype=tf.uint8)
        if height is not None and width is not None:
            invalid_masks = uflow_utils.resize(invalid_masks,
                                               height,
                                               width,
                                               is_flow=False)
        output.append(invalid_masks)

    # Only put the output in a list if there are more than one items in there.
    if len(output) == 1:
        output = output[0]

    # print("output.shape: ", output.shape)
    return output
Esempio n. 38
0
def _conditional(Xnew,
                 feat,
                 kern,
                 f,
                 *,
                 full_cov=False,
                 full_output_cov=False,
                 q_sqrt=None,
                 white=False):
    """
    Multi-output GP with independent GP priors.
    Number of latent processes equals the number of outputs (L = P).

    The covariance matrices used to calculate the conditional have the following shape:
    - Kuu: P x M x M
    - Kuf: P x M x N
    - Kff: P x N or P x N x N

    Further reference
    -----------------
    - See `gpflow.conditionals._conditional` for a detailed explanation of
      conditional in the single-output case.
    - See the multiouput notebook for more information about the multiouput framework.
    - See above for the parameters and the return value.
    """

    logger.debug(
        "conditional: object, SharedIndependentMof, SeparateIndependentMok, object"
    )
    # Following are: P x M x M  -  P x M x N  -  P x N(x N)
    Kmms = Kuu(feat, kern, jitter=settings.numerics.jitter_level)  # P x M x M
    Kmns = Kuf(feat, kern, Xnew)  # P x M x N
    kern_list = kern.kernels if isinstance(
        kern, Combination) else [kern.kern] * len(feat.feat_list)
    Knns = tf.stack(
        [k.K(Xnew) if full_cov else k.Kdiag(Xnew) for k in kern_list], axis=0)
    fs = tf.transpose(f)[:, :, None]  # P x M x 1
    # P x 1 x M x M  or  P x M x 1
    q_sqrts = tf.transpose(
        q_sqrt)[:, :, None] if q_sqrt.shape.ndims == 2 else q_sqrt[:,
                                                                   None, :, :]

    def single_gp_conditional(t):
        Kmm, Kmn, Knn, f, q_sqrt = t
        return base_conditional(Kmn,
                                Kmm,
                                Knn,
                                f,
                                full_cov=full_cov,
                                q_sqrt=q_sqrt,
                                white=white)

    rmu, rvar = tf.map_fn(
        single_gp_conditional, (Kmms, Kmns, Knns, fs, q_sqrts),
        (settings.float_type,
         settings.float_type))  # P x N x 1, P x 1 x N x N or P x N x 1

    fmu = tf.matrix_transpose(rmu[:, :, 0])  # N x P

    if full_cov:
        fvar = rvar[:, 0, :, :]  # P x N x N
    else:
        fvar = tf.transpose(rvar[..., 0])  # N x P

    return fmu, _expand_independent_outputs(fvar, full_cov, full_output_cov)
Esempio n. 39
0
    def conditional(self, X, test, full_cov=False):
        """
        A multisample conditional, where X is shape (S,N,D_out), independent over samples S

        if full_cov is True
            mean is (S,N,D_out), var is (S,N,N,D_out)

        if full_cov is False
            mean and var are both (S,N,D_out)

        :param X:  The input locations (S,N,D_in)
        :param full_cov: Whether to calculate full covariance or just diagonal
        :return: mean (S,N,D_out), var (S,N,D_out or S,N,N,D_out)
        """
        def single_sample_conditional(X, full_cov=False):

            if test == 0:

                select = tf.random_shuffle(tf.range(tf.shape(
                    self.q_mu)[1]))[:tf.cast(
                        (1.0 - self.dropout) *
                        tf.cast(tf.shape(self.q_mu)[1], tf.float64), tf.int32)]
                select = tf.contrib.framework.sort(select)
                q_mu_temp = tf.gather(self.q_mu, select, axis=1)
                q_sqrt_temp = tf.gather(self.q_sqrt, select, axis=0)
                '''
                select  = np.random.choice((tf.convert_to_tensor(self.q_mu.shape[1])).eval()-1, size=int((1-self.dropout)*((tf.convert_to_tensor(self.q_mu.shape[1])).eval()-1)), replace = False)
                select = np.sort(select)
                q_mu_temp = np.take((self.q_mu).eval(),select,axis=1)
                q_sqrt_temp = np.take((self.q_sqrt).eval(),select,axis=0)
                transform = transforms.LowerTriangular((tf.convert_to_tensor(self.feature.Z.shape[0])).eval(), num_matrices=q_mu_temp.shape[1])
                q_sqrt_temp = Parameter(q_sqrt_temp, transform=transform)
                q_mu_temp = tf.constant(q_mu_temp)
                q_sqrt_temp = tf.constant(q_sqrt_temp)
                '''

            else:
                q_mu_temp = self.q_mu
                q_sqrt_temp = self.q_sqrt
                select = []

            self.q_mu_temp = q_mu_temp
            self.q_sqrt_temp = q_sqrt_temp

            mean, var = my_conditional(X,
                                       self.feature.Z,
                                       self.kern,
                                       q_mu_temp,
                                       q_sqrt=q_sqrt_temp,
                                       full_cov=full_cov,
                                       white=True)
            return mean + self.mean_function(X), var, select

        if full_cov is True:
            f = lambda a: single_sample_conditional(a, full_cov=full_cov)
            mean, var, select = tf.map_fn(f, X, dtype=(tf.float64, tf.float64))
            return tf.stack(mean), tf.stack(var), select
        else:
            S, N, D = tf.shape(X)[0], tf.shape(X)[1], tf.shape(X)[2]
            X_flat = tf.reshape(X, [S * N, D])
            mean, var, select = single_sample_conditional(X_flat)
            mean = tf.reshape(mean, [S, N, -1])
            var = tf.reshape(var, [S, N, -1])
            return mean, var, select
Esempio n. 40
0
def draw_bounding_boxes_on_image_tensors(images,
                                         boxes,
                                         classes,
                                         scores,
                                         category_index,
                                         instance_masks=None,
                                         keypoints=None,
                                         max_boxes_to_draw=20,
                                         min_score_thresh=0.2,
                                         use_normalized_coordinates=True):
    """Draws bounding boxes, masks, and keypoints on batch of image tensors.

  Args:
    images: A 4D uint8 image tensor of shape [N, H, W, C]. If C > 3, additional
      channels will be ignored.
    boxes: [N, max_detections, 4] float32 tensor of detection boxes.
    classes: [N, max_detections] int tensor of detection classes. Note that
      classes are 1-indexed.
    scores: [N, max_detections] float32 tensor of detection scores.
    category_index: a dict that maps integer ids to category dicts. e.g.
      {1: {1: 'dog'}, 2: {2: 'cat'}, ...}
    instance_masks: A 4D uint8 tensor of shape [N, max_detection, H, W] with
      instance masks.
    keypoints: A 4D float32 tensor of shape [N, max_detection, num_keypoints, 2]
      with keypoints.
    max_boxes_to_draw: Maximum number of boxes to draw on an image. Default 20.
    min_score_thresh: Minimum score threshold for visualization. Default 0.2.
    use_normalized_coordinates: Whether to assume boxes and kepoints are in
      normalized coordinates (as opposed to absolute coordiantes).
      Default is True.

  Returns:
    4D image tensor of type uint8, with boxes drawn on top.
  """
    # Additional channels are being ignored.
    images = images[:, :, :, 0:3]
    visualization_keyword_args = {
        'use_normalized_coordinates': use_normalized_coordinates,
        'max_boxes_to_draw': max_boxes_to_draw,
        'min_score_thresh': min_score_thresh,
        'agnostic_mode': False,
        'line_thickness': 4
    }

    if instance_masks is not None and keypoints is None:
        visualize_boxes_fn = functools.partial(_visualize_boxes_and_masks,
                                               category_index=category_index,
                                               **visualization_keyword_args)
        elems = [images, boxes, classes, scores, instance_masks]
    elif instance_masks is None and keypoints is not None:
        visualize_boxes_fn = functools.partial(_visualize_boxes_and_keypoints,
                                               category_index=category_index,
                                               **visualization_keyword_args)
        elems = [images, boxes, classes, scores, keypoints]
    elif instance_masks is not None and keypoints is not None:
        visualize_boxes_fn = functools.partial(
            _visualize_boxes_and_masks_and_keypoints,
            category_index=category_index,
            **visualization_keyword_args)
        elems = [images, boxes, classes, scores, instance_masks, keypoints]
    else:
        visualize_boxes_fn = functools.partial(_visualize_boxes,
                                               category_index=category_index,
                                               **visualization_keyword_args)
        elems = [images, boxes, classes, scores]

    def draw_boxes(image_and_detections):
        """Draws boxes on image."""
        image_with_boxes = tf.py_func(visualize_boxes_fn, image_and_detections,
                                      tf.uint8)
        return image_with_boxes

    images = tf.map_fn(draw_boxes, elems, dtype=tf.uint8, back_prop=False)
    return images
Esempio n. 41
0
    def decode(self, knowledge_rep, dropout=None):
        """
        takes in a knowledge representation
        and output a probability estimation over
        all paragraph tokens on which token should be
        the start of the answer span, and which should be
        the end of the answer span.

        :param knowledge_rep: it is a representation of the paragraph and question,
                              decided by how you choose to implement the encoder
        :return:
        """

        # Approach 1

        # Run final Uniderectional LSTM one final inputs representation
        #finalInputs, contextLen = knowledge_rep
        #with vs.variable_scope("FinalLSTMs", reuse = False):
            #outputs_s, _ = tf.nn.dynamic_rnn(self.LSTMcellCompresser, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)
        #    _, (_, output_s) = tf.nn.dynamic_rnn(self.LSTMcell, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)

        #with vs.variable_scope("FinalLSTMe", reuse=False):
            #outputs_e, _ = tf.nn.dynamic_rnn(self.LSTMcellCompresser, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)
        #    _, (_, output_e) = tf.nn.dynamic_rnn(self.LSTMcell, inputs=finalInputs, sequence_length=contextLen, dtype=tf.float32)

        #return(output_s, output_e)
        #outputs_s = outputs[:, :, 0]
        #outputs_e = outputs[:, :, 1]
        #outputs_s = tf.reshape(outputs_s, [-1, self.output_size])
        #outputs_e = tf.reshape(outputs_e, [-1, self.output_size])

        #return(outputs_s, outputs_e)


        # Approach 2
        #final_inputs, contextLen = knowledge_rep
        #with vs.variable_scope("Decoder", reuse=False):
        #    output_s = tf.contrib.layers.fully_connected(inputs=final_inputs, num_outputs=self.output_size,
        #                                                 weights_initializer=tf.contrib.layers.xavier_initializer())
        #    output_e = tf.contrib.layers.fully_connected(inputs=final_inputs, num_outputs=self.output_size,
        #                                                 weights_initializer=tf.contrib.layers.xavier_initializer())
        #return (output_s, output_e)



        # Approach 3
        #_, max_length, encoded_size = knowledge_rep.get_shape().as_list()
        final_inputs, contextLen = knowledge_rep

        with vs.variable_scope("Decoder", reuse=None):
            W_s = tf.get_variable("W_s", shape=(final_inputs.get_shape()[2], 1), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())
            W_e = tf.get_variable("W_e", shape=(final_inputs.get_shape()[2], 1), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())

            tmp_s = tf.map_fn(lambda current_output: tf.matmul(current_output, W_s), final_inputs)
            tmp_e = tf.map_fn(lambda current_output: tf.matmul(current_output, W_e), final_inputs)

            #tmp_s = tf.map_fn(lambda current_output: tf.matmul(tf.nn.dropout(current_output, dropout), W_s), final_inputs)
            #tmp_e = tf.map_fn(lambda current_output: tf.matmul(tf.nn.dropout(current_output, dropout), W_e), final_inputs)

            outputs_s = tf.reshape(tmp_s, [-1, self.output_size])
            outputs_e = tf.reshape(tmp_e, [-1, self.output_size])

        return(outputs_s, outputs_e)
    def call(self, y_pred, mask=None):
        '''
        Returns:
            3D tensor of shape `(batch_size, top_k, 6)`. The second axis is
            zero-padded
            to always yield `top_k` predictions per batch item. The last
            axis contains
            the coordinates for each predicted box in the format
            `[class_id, confidence, xmin, ymin, xmax, ymax]`.
        '''

        #######################################################################
        # 1. Convert the box coordinates from predicted anchor box offsets
        # to predicted
        #    absolute coordinates
        #######################################################################

        # Convert anchor box offsets to image offsets.
        cx = y_pred[..., -12] * y_pred[..., -4] * y_pred[..., -6] + y_pred[
            ..., -8]  # cx = cx_pred * cx_variance * w_anchor + cx_anchor
        cy = y_pred[..., -11] * y_pred[..., -3] * y_pred[..., -5] + y_pred[
            ..., -7]  # cy = cy_pred * cy_variance * h_anchor + cy_anchor
        w = tf.exp(y_pred[..., -10] * y_pred[..., -2]) * y_pred[
            ..., -6]  # w = exp(w_pred * variance_w) * w_anchor
        h = tf.exp(y_pred[..., -9] * y_pred[..., -1]) * y_pred[
            ..., -5]  # h = exp(h_pred * variance_h) * h_anchor

        # Convert 'centroids' to 'corners'.
        xmin = cx - 0.5 * w
        ymin = cy - 0.5 * h
        xmax = cx + 0.5 * w
        ymax = cy + 0.5 * h

        # If the model predicts box coordinates relative to the image
        # dimensions and they are supposed
        # to be converted back to absolute coordinates, do that.
        def normalized_coords():
            xmin1 = tf.expand_dims(xmin * self.tf_img_width, axis=-1)
            ymin1 = tf.expand_dims(ymin * self.tf_img_height, axis=-1)
            xmax1 = tf.expand_dims(xmax * self.tf_img_width, axis=-1)
            ymax1 = tf.expand_dims(ymax * self.tf_img_height, axis=-1)
            return xmin1, ymin1, xmax1, ymax1

        def non_normalized_coords():
            return tf.expand_dims(xmin, axis=-1), tf.expand_dims(ymin,
                                                                 axis=-1), \
                   tf.expand_dims(
                       xmax, axis=-1), tf.expand_dims(ymax, axis=-1)

        xmin, ymin, xmax, ymax = tf.cond(self.tf_normalize_coords,
                                         normalized_coords,
                                         non_normalized_coords)

        # Concatenate the one-hot class confidences and the converted box
        # coordinates to form the decoded predictions tensor.
        y_pred = tf.concat(values=[y_pred[..., :-12], xmin, ymin, xmax, ymax],
                           axis=-1)

        #######################################################################
        # 2. Perform confidence thresholding, per-class non-maximum
        # suppression, and
        #    top-k filtering.
        #######################################################################

        batch_size = tf.shape(y_pred)[0]  # Output dtype: tf.int32
        n_boxes = tf.shape(y_pred)[1]
        n_classes = y_pred.shape[2] - 4
        class_indices = tf.range(1, n_classes)

        # Create a function that filters the predictions for the given batch
        #  item. Specifically, it performs:
        # - confidence thresholding
        # - non-maximum suppression (NMS)
        # - top-k filtering
        def filter_predictions(batch_item):
            # Create a function that filters the predictions for one single
            # class.
            def filter_single_class(index):
                # From a tensor of shape (n_boxes, n_classes + 4
                # coordinates) extract
                # a tensor of shape (n_boxes, 1 + 4 coordinates) that
                # contains the
                # confidnece values for just one class, determined by `index`.
                confidences = tf.expand_dims(batch_item[..., index], axis=-1)
                class_id = tf.fill(dims=tf.shape(confidences),
                                   value=tf.to_float(index))
                box_coordinates = batch_item[..., -4:]

                single_class = tf.concat(
                    [class_id, confidences, box_coordinates], axis=-1)

                # Apply confidence thresholding with respect to the class
                # defined by `index`.
                threshold_met = single_class[:, 1] > self.tf_confidence_thresh
                single_class = tf.boolean_mask(tensor=single_class,
                                               mask=threshold_met)

                # If any boxes made the threshold, perform NMS.
                def perform_nms():
                    scores = single_class[..., 1]

                    # `tf.image.non_max_suppression()` needs the box
                    # coordinates in the format `(ymin, xmin, ymax, xmax)`.
                    xmin = tf.expand_dims(single_class[..., -4], axis=-1)
                    ymin = tf.expand_dims(single_class[..., -3], axis=-1)
                    xmax = tf.expand_dims(single_class[..., -2], axis=-1)
                    ymax = tf.expand_dims(single_class[..., -1], axis=-1)
                    boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)

                    maxima_indices = tf.image.non_max_suppression(
                        boxes=boxes,
                        scores=scores,
                        max_output_size=self.tf_nms_max_output_size,
                        iou_threshold=self.iou_threshold,
                        name='non_maximum_suppresion')
                    maxima = tf.gather(params=single_class,
                                       indices=maxima_indices,
                                       axis=0)
                    return maxima

                def no_confident_predictions():
                    return tf.constant(value=0.0, shape=(1, 6))

                single_class_nms = tf.cond(tf.equal(tf.size(single_class), 0),
                                           no_confident_predictions,
                                           perform_nms)

                # Make sure `single_class` is exactly
                # `self.nms_max_output_size` elements long.
                padded_single_class = tf.pad(tensor=single_class_nms,
                                             paddings=[[
                                                 0,
                                                 self.tf_nms_max_output_size -
                                                 tf.shape(single_class_nms)[0]
                                             ], [0, 0]],
                                             mode='CONSTANT',
                                             constant_values=0.0)

                return padded_single_class

            # Iterate `filter_single_class()` over all class indices.
            filtered_single_classes = tf.map_fn(
                fn=lambda i: filter_single_class(i),
                elems=tf.range(1, n_classes),
                dtype=tf.float32,
                parallel_iterations=128,
                back_prop=False,
                swap_memory=False,
                infer_shape=True,
                name='loop_over_classes')

            # Concatenate the filtered results for all individual classes to
            #  one tensor.
            filtered_predictions = tf.reshape(tensor=filtered_single_classes,
                                              shape=(-1, 6))

            # Perform top-k filtering for this batch item or pad it in case
            # there are
            # fewer than `self.top_k` boxes left at this point. Either way,
            # produce a
            # tensor of length `self.top_k`. By the time we return the final
            #  results tensor
            # for the whole batch, all batch items must have the same number
            #  of predicted
            # boxes so that the tensor dimensions are homogenous. If fewer
            # than `self.top_k`
            # predictions are left after the filtering process above,
            # we pad the missing
            # predictions with zeros as dummy entries.
            def top_k():
                return tf.gather(params=filtered_predictions,
                                 indices=tf.nn.top_k(filtered_predictions[:,
                                                                          1],
                                                     k=self.tf_top_k,
                                                     sorted=True).indices,
                                 axis=0)

            def pad_and_top_k():
                padded_predictions = tf.pad(
                    tensor=filtered_predictions,
                    paddings=[[
                        0, self.tf_top_k - tf.shape(filtered_predictions)[0]
                    ], [0, 0]],
                    mode='CONSTANT',
                    constant_values=0.0)
                return tf.gather(params=padded_predictions,
                                 indices=tf.nn.top_k(padded_predictions[:, 1],
                                                     k=self.tf_top_k,
                                                     sorted=True).indices,
                                 axis=0)

            top_k_boxes = tf.cond(
                tf.greater_equal(
                    tf.shape(filtered_predictions)[0], self.tf_top_k), top_k,
                pad_and_top_k)

            return top_k_boxes

        # Iterate `filter_predictions()` over all batch items.
        output_tensor = tf.map_fn(fn=lambda x: filter_predictions(x),
                                  elems=y_pred,
                                  dtype=None,
                                  parallel_iterations=128,
                                  back_prop=False,
                                  swap_memory=False,
                                  infer_shape=True,
                                  name='loop_over_batch')

        return output_tensor
Esempio n. 43
0
################ inputs do grafo ##################
tf_x_input = tf.placeholder(tf.float32, [None, 40, 140], name='X_input')
tf_X = tf.reshape(tf_x_input, shape=[
    -1, 40, 140, 1
])  # reformata input para forma de imagem requerida pelo tf (h, w, d)

tf_y_input = tf.placeholder(tf.int64, [None, seq_len, n_classes],
                            name='y_input')
tf_keep_prob = tf.placeholder(tf.float32)  # probabilidade de drop_out no grafo

# augment dataset if training
tf_X = tf.cond(
    tf.less_equal(tf_keep_prob,
                  0.98),  # usa prob de dropout para saber se está treinando
    lambda: tf.map_fn(lambda image: augment(image), tf_X
                      ),  # caso treinando, aumenta os dados
    lambda: tf.map_fn(lambda image: center_crop(image), tf_X)
)  # caso testanto, corta imagen no centro

################ Variáveis do grafo ################
# Inicialização Xavier dos parâmetros
init_wc1 = np.sqrt(6.0 / (img_w * img_h * 1 + img_w * img_h * CL1_depth))
init_wc2 = np.sqrt(
    6.0 / (img_w * img_h * CL1_depth + img_w / 2 * img_h / 2 * CL2_depth))
init_wc3 = np.sqrt(
    6.0 /
    (img_w / 2 * img_h / 2 * CL2_depth + img_w / 4 * img_h / 4 * CL3_depth))
init_wc4 = np.sqrt(
    6.0 /
    (img_w / 4 * img_h / 4 * CL3_depth + img_w / 8 * img_h / 8 * CL4_depth))
init_wfc1 = np.sqrt(6.0 / (img_w / 8 * img_h / 8 * CL4_depth + DL1_size))
Esempio n. 44
0
# Weight for output layers
with tf.name_scope("linear_layer_weights"):
    with tf.name_scope("W_linear"):
        Wl = tf.Variable(
            tf.truncated_normal([hidden_layer_size, num_classes],
                                mean=0,
                                stddev=.01))
        variable_summaries(Wl)
    with tf.name_scope("Bias_linear"):
        bl = tf.Variable(tf.truncated_normal([num_classes], mean=0,
                                             stddev=.01))
        variable_summaries(bl)

with tf.name_scope("linear_layer_weights"):
    #Iterate accross time and apply linear layer to all RNN outputs
    all_outputs = tf.map_fn(get_linear_layer, all_hidden_states)
    # Get last output
    output = all_outputs[-1]
    tf.summary.histogram("outputs", output)

with tf.name_scope("cross_entropy"):
    cross_entropy = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y))
    tf.summary.scalar("cross_entropy", cross_entropy)

with tf.name_scope("train"):
    # Uses RMSPropOptimizer
    # Minimize the error
    train_step = tf.train.RMSPropOptimizer(1e-3, 0.9).minimize(cross_entropy)

with tf.name_scope("accuracy"):
Esempio n. 45
0
def train(x, y, num_epochs, batch_size, graph_type='regular'):
    """
    Train a specified network architecture on a dataset
    :param x: Inputs
    :param y: Labels
    :param num_epochs: Passes through data
    :param batch_size: Mini batch size
    :param graph_type: Network architecture
    :return:
    """

    train_dataset = tf.data.Dataset.from_tensor_slices((x, y))
    train_dataset = train_dataset.shuffle(x.shape[0])
    train_dataset = train_dataset.repeat(num_epochs)
    train_dataset = train_dataset.batch(batch_size)
    iterator = train_dataset.make_initializable_iterator()

    inputs, labels = iterator.get_next()
    inputs = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame),
                       inputs,
                       dtype=tf.float32)

    if graph_type == 'regular':
        logits = regular_net(inputs, 10)
    elif graph_type == 'tf':
        logits = weightnorm_net(inputs, 10)
    elif graph_type == 'keras':
        logits = weightnorm_keras_net(inputs, 10)
    else:
        raise ValueError

    loss_op = tf.losses.sparse_softmax_cross_entropy(logits=logits,
                                                     labels=labels)

    optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
                                           momentum=momentum)

    train_op = optimizer.minimize(loss_op)
    init = tf.global_variables_initializer()

    step = 0
    running_loss = 0
    running_loss_array = []

    with tf.Session() as sess:
        sess.run(init)
        sess.run(iterator.initializer)

        graph_path = os.path.join(os.getcwd(), 'train', graph_type)
        tf.summary.FileWriter(graph_path, sess.graph)

        while True:
            try:
                _, loss = sess.run([train_op, loss_op])
                step += 1
                running_loss += (loss / batch_size)
                if step % 32 == (32 - 1):
                    running_loss_array.append(running_loss)
                    running_loss = 0.0

            except tf.errors.OutOfRangeError:
                return running_loss_array
Esempio n. 46
0
    def call(self, y_pred, mask=None):
        '''
        Returns:
            3D tensor of shape `(batch_size, top_k, 6)`. The second axis is zero-padded
            to always yield `top_k` predictions per batch item. The last axis contains
            the coordinates for each predicted box in the format
            `[class_id, confidence, xmin, ymin, xmax, ymax]`.
        '''

        #####################################################################################
        # 1. Convert the box coordinates from predicted anchor box offsets to predicted
        #    absolute coordinates
        #####################################################################################

        # Extract the predicted class IDs as the indices of the highest confidence values.
        class_ids = tf.expand_dims(tf.to_float(
            tf.argmax(y_pred[..., :-12], axis=-1)),
                                   axis=-1)
        # Extract the confidences of the maximal classes.
        confidences = tf.reduce_max(y_pred[..., :-12], axis=-1, keep_dims=True)

        # Convert anchor box offsets to image offsets.
        cx = y_pred[..., -12] * y_pred[..., -4] * y_pred[..., -6] + y_pred[
            ..., -8]  # cx = cx_pred * cx_variance * w_anchor + cx_anchor
        cy = y_pred[..., -11] * y_pred[..., -3] * y_pred[..., -5] + y_pred[
            ..., -7]  # cy = cy_pred * cy_variance * h_anchor + cy_anchor
        w = tf.exp(y_pred[..., -10] * y_pred[..., -2]) * y_pred[
            ..., -6]  # w = exp(w_pred * variance_w) * w_anchor
        h = tf.exp(y_pred[..., -9] * y_pred[..., -1]) * y_pred[
            ..., -5]  # h = exp(h_pred * variance_h) * h_anchor

        # Convert 'centroids' to 'corners'.
        xmin = cx - 0.5 * w
        ymin = cy - 0.5 * h
        xmax = cx + 0.5 * w
        ymax = cy + 0.5 * h

        # If the model predicts box coordinates relative to the image dimensions and they are supposed
        # to be converted back to absolute coordinates, do that.
        def normalized_coords():
            xmin1 = tf.expand_dims(xmin * self.tf_img_width, axis=-1)
            ymin1 = tf.expand_dims(ymin * self.tf_img_height, axis=-1)
            xmax1 = tf.expand_dims(xmax * self.tf_img_width, axis=-1)
            ymax1 = tf.expand_dims(ymax * self.tf_img_height, axis=-1)
            return xmin1, ymin1, xmax1, ymax1

        def non_normalized_coords():
            return tf.expand_dims(xmin, axis=-1), tf.expand_dims(
                ymin,
                axis=-1), tf.expand_dims(xmax,
                                         axis=-1), tf.expand_dims(ymax,
                                                                  axis=-1)

        xmin, ymin, xmax, ymax = tf.cond(self.tf_normalize_coords,
                                         normalized_coords,
                                         non_normalized_coords)

        # Concatenate the one-hot class confidences and the converted box coordinates to form the decoded predictions tensor.
        y_pred = tf.concat(
            values=[class_ids, confidences, xmin, ymin, xmax, ymax], axis=-1)

        #####################################################################################
        # 2. Perform confidence thresholding, non-maximum suppression, and top-k filtering.
        #####################################################################################

        batch_size = tf.shape(y_pred)[0]  # Output dtype: tf.int32
        n_boxes = tf.shape(y_pred)[1]
        n_classes = y_pred.shape[2] - 4
        class_indices = tf.range(1, n_classes)

        # Create a function that filters the predictions for the given batch item. Specifically, it performs:
        # - confidence thresholding
        # - non-maximum suppression (NMS)
        # - top-k filtering
        def filter_predictions(batch_item):

            # Keep only the non-background boxes.
            positive_boxes = tf.not_equal(batch_item[..., 0], 0.0)
            predictions = tf.boolean_mask(tensor=batch_item,
                                          mask=positive_boxes)

            def perform_confidence_thresholding():
                # Apply confidence thresholding.
                threshold_met = predictions[:, 1] > self.tf_confidence_thresh
                return tf.boolean_mask(tensor=predictions, mask=threshold_met)

            def no_positive_boxes():
                return tf.constant(value=0.0, shape=(1, 6))

            # If there are any positive predictions, perform confidence thresholding.
            predictions_conf_thresh = tf.cond(
                tf.equal(tf.size(predictions), 0), no_positive_boxes,
                perform_confidence_thresholding)

            def perform_nms():
                scores = predictions_conf_thresh[..., 1]

                # `tf.image.non_max_suppression()` needs the box coordinates in the format `(ymin, xmin, ymax, xmax)`.
                xmin = tf.expand_dims(predictions_conf_thresh[..., -4],
                                      axis=-1)
                ymin = tf.expand_dims(predictions_conf_thresh[..., -3],
                                      axis=-1)
                xmax = tf.expand_dims(predictions_conf_thresh[..., -2],
                                      axis=-1)
                ymax = tf.expand_dims(predictions_conf_thresh[..., -1],
                                      axis=-1)
                boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)

                maxima_indices = tf.image.non_max_suppression(
                    boxes=boxes,
                    scores=scores,
                    max_output_size=self.tf_nms_max_output_size,
                    iou_threshold=self.iou_threshold,
                    name='non_maximum_suppresion')
                maxima = tf.gather(params=predictions_conf_thresh,
                                   indices=maxima_indices,
                                   axis=0)
                return maxima

            def no_confident_predictions():
                return tf.constant(value=0.0, shape=(1, 6))

            # If any boxes made the threshold, perform NMS.
            predictions_nms = tf.cond(
                tf.equal(tf.size(predictions_conf_thresh), 0),
                no_confident_predictions, perform_nms)

            # Perform top-k filtering for this batch item or pad it in case there are
            # fewer than `self.top_k` boxes left at this point. Either way, produce a
            # tensor of length `self.top_k`. By the time we return the final results tensor
            # for the whole batch, all batch items must have the same number of predicted
            # boxes so that the tensor dimensions are homogenous. If fewer than `self.top_k`
            # predictions are left after the filtering process above, we pad the missing
            # predictions with zeros as dummy entries.
            def top_k():
                return tf.gather(params=predictions_nms,
                                 indices=tf.nn.top_k(predictions_nms[:, 1],
                                                     k=self.tf_top_k,
                                                     sorted=True).indices,
                                 axis=0)

            def pad_and_top_k():
                padded_predictions = tf.pad(tensor=predictions_nms,
                                            paddings=[[
                                                0, self.tf_top_k -
                                                tf.shape(predictions_nms)[0]
                                            ], [0, 0]],
                                            mode='CONSTANT',
                                            constant_values=0.0)
                return tf.gather(params=padded_predictions,
                                 indices=tf.nn.top_k(padded_predictions[:, 1],
                                                     k=self.tf_top_k,
                                                     sorted=True).indices,
                                 axis=0)

            top_k_boxes = tf.cond(
                tf.greater_equal(tf.shape(predictions_nms)[0], self.tf_top_k),
                top_k, pad_and_top_k)

            return top_k_boxes

        # Iterate `filter_predictions()` over all batch items.
        output_tensor = tf.map_fn(fn=lambda x: filter_predictions(x),
                                  elems=y_pred,
                                  dtype=None,
                                  parallel_iterations=128,
                                  back_prop=False,
                                  swap_memory=False,
                                  infer_shape=True,
                                  name='loop_over_batch')

        return output_tensor
Esempio n. 47
0
def test_lanenet(dataset_dir, image_path, weights_path, use_gpu, image_list, batch_size, save_dir):

    """
    :param image_path:
    :param weights_path:
    :param use_gpu:
    :return:
    """
    
    test_dataset = lanenet_data_processor_test.DataSet(image_path, batch_size, dataset_dir)
    input_tensor = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensor')
    imgs = tf.map_fn(test_dataset.process_img, input_tensor, dtype=tf.float32)
    phase_tensor = tf.constant('test', tf.string)

    net = lanenet_merge_model.LaneNet()
    binary_seg_ret, instance_seg_ret = net.test_inference(imgs, phase_tensor, 'lanenet_loss')
    initial_var = tf.global_variables()
    final_var = initial_var[:-1]
    saver = tf.train.Saver(final_var)
    # Set sess configuration
    if use_gpu:
        sess_config = tf.ConfigProto(device_count={'GPU': 1})
    else:
        sess_config = tf.ConfigProto(device_count={'GPU': 0})
    sess_config.gpu_options.per_process_gpu_memory_fraction = CFG.TEST.GPU_MEMORY_FRACTION
    sess_config.gpu_options.allow_growth = CFG.TRAIN.TF_ALLOW_GROWTH
    sess_config.gpu_options.allocator_type = 'BFC'
    sess = tf.Session(config=sess_config)
    with sess.as_default():
        sess.run(tf.global_variables_initializer())
        saver.restore(sess=sess, save_path=weights_path)
        for i in range(math.ceil(len(image_list) / batch_size)):
            print("Inferring batch {} with batch size {}".format(i, batch_size))
            paths = test_dataset.next_batch()
            instance_seg_image, existence_output = sess.run([binary_seg_ret, instance_seg_ret],
                                                            feed_dict={input_tensor: paths})
            # pudb.set_trace()
            for cnt, input_image_path in enumerate(paths):
                print("Generating output for input image {}".format(input_image_path))
                input_image = cv2.imread(input_image_path)
                input_image = cv2.resize(input_image, (CFG.TRAIN.IMG_WIDTH, CFG.TRAIN.IMG_HEIGHT))

                all_lanes_image = np.zeros((CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH))

                parent_path = os.path.dirname(input_image_path)
                output_dir = os.path.join(parent_path, save_dir)
                if not os.path.exists(output_dir):
                    os.makedirs(output_dir)

                file_exist = open(os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'exist.txt'), 'w')
                for cnt_img in range(4):
                    filename = os.path.join(output_dir, os.path.basename(input_image_path)[:-4] + '_' + str(cnt_img + 1) + '_avg.png')
                    lane_image = (instance_seg_image[cnt, :, :, cnt_img + 1] * 255).astype(int)
                    cv2.imwrite(filename, lane_image)
                    all_lanes_image = all_lanes_image + lane_image
                    if existence_output[cnt, cnt_img] > 0.5:
                        file_exist.write('1 ')
                    else:
                        file_exist.write('0 ')
                file_exist.close()

                # Make all_lanes_image into color image (3 channels)
                all_lanes_color_image = np.zeros((CFG.TRAIN.IMG_HEIGHT, CFG.TRAIN.IMG_WIDTH, 3))
                all_lanes_color_image[:, :, 2] = all_lanes_image[:, :]

                all_lanes_color_image_path = os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'lanes.jpg')
                resized_input_image_path = os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'resized_input.jpg')
                overlay_image_path = os.path.join(output_dir, os.path.basename(input_image_path)[:-3] + 'overlay.jpg')

                cv2.imwrite(resized_input_image_path, input_image)
                cv2.imwrite(all_lanes_color_image_path, all_lanes_color_image)
                cv2.imwrite(overlay_image_path, all_lanes_color_image + input_image)


    sess.close()
    return
    def __init__(self, models_path,  vehicle_name='drone0'):
        self.g = tf.Graph()
        self.vehicle_name = vehicle_name
        self.iter=0
        self.models_path = models_path
        self.first_frame = True
        self.last_frame = []
        with self.g.as_default():
            # stat_writer_path = './adda/return_plot/'
            # loss_writer_path = './adda/return_plot/'+ self.vehicle_name + '/loss/'
            # self.stat_writer = tf.summary.FileWriter(stat_writer_path)
            # name_array = 'D:/train/loss'+'/'+name
            # self.loss_writer = tf.summary.FileWriter(loss_writer_path)


            self.batch_size = tf.placeholder(tf.int32, shape=())
            self.learning_rate = tf.placeholder(tf.float32, shape=())
            self.X1 = tf.placeholder(tf.float32, [None, 103,103, 3], name='States')
            self.X1T = tf.placeholder(tf.float32, [None, 103,103, 3], name='StatesT')


            self.X = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.X1)
            self.XT = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.X1T)
            # self.X = tf.map_fn(lambda frame: tf.image.per_image_standardization(frame), self.X1)
            self.Y = tf.placeholder(tf.float32, shape=[None], name='Qvals')
            self.YT = tf.placeholder(tf.float32, shape=[None], name='Qvals')
            self.Yactions = tf.placeholder(tf.int32, shape=[None], name='Actions')
            self.YTactions = tf.placeholder(tf.int32, shape=[None], name='Actions')

            # create graph
            nn = adda.ADDA(25)
            # for source domain
            self.feat_s = nn.s_encoder(self.X, reuse=False, trainable=False)
            self.logits_s = nn.classifier(self.feat_s, reuse=False, trainable=False)
            self.disc_s = nn.discriminator(self.feat_s, reuse=False)

            # for target domain
            self.feat_t = nn.t_encoder(self.XT, reuse=False)
            self.logits_t = nn.classifier(self.feat_t, reuse=True, trainable=False)
            self.disc_t = nn.discriminator(self.feat_t, reuse=True)

            # build loss
            self.g_loss, self.d_loss = nn.build_ad_loss(self.disc_s, self.disc_t)
            # g_loss,d_loss = nn.build_w_loss(disc_s,disc_t)

            # create optimizer for two task
            self.var_t_en = tf.trainable_variables(nn.t_e)
            self.optim_g = tf.train.AdamOptimizer(0.00001, beta1=0.5, beta2=0.999).minimize(self.g_loss, var_list=self.var_t_en)

            self.var_d = tf.trainable_variables(nn.d)
            self.optim_d = tf.train.AdamOptimizer(0.00001, beta1=0.5, beta2=0.999).minimize(self.d_loss, var_list=self.var_d)






            # create acuuracy op with training batch
            self.acc_tr_s = nn.eval( self.logits_s, self.Yactions)
            self.acc_tr_t = nn.eval( self.logits_t, self.YTactions)






            # create source saver for restore s_encoder
            encoder_path = tf.train.latest_checkpoint(models_path + "/encoder/")
            classifier_path = tf.train.latest_checkpoint(models_path + "/classifier/")

            if encoder_path is None:
                raise ValueError("Don't exits in this dir")
            if classifier_path is None:
                raise ValueError("Don't exits in this dir")


            self.source_var = tf.contrib.framework.list_variables(encoder_path)

            self.var_s_g = tf.global_variables(scope=nn.s_e)
            self.var_c_g = tf.global_variables(scope=nn.c)
            self. var_t_g = tf.trainable_variables(scope=nn.t_e)

            self.encoder_saver = tf.train.Saver(var_list=self.var_s_g)
            self.classifier_saver = tf.train.Saver(var_list=self.var_c_g)

            self.encoder_saver_target = tf.train.Saver(var_list=self.var_s_g)
            dict_var = {}
            for i in self.source_var:
                for j in self.var_t_g:
                    if i[0][1:] in j.name[1:]:
                        dict_var[i[0]] = j
                        # print(dict_var)
            self.fine_turn_saver = tf.train.Saver(var_list=dict_var)


            # dict_var2 = {}
            # for i in self.var_t_g:
            #     for j in self.source_var:
            #         if i[0][1:] in j.name[1:]:
            #             dict_var[i[0]] = j
                        # print(dict_var)
            # self.fine_turn_saver2= tf.train.Saver(var_list=dict_var2)
            # self.fine_turn_saver = tf.train.Saver(var_list=dict_var)
            # assert False
            # create this model saver
            self.best_saver = tf.train.Saver(max_to_keep=3)

            self.sess = tf.InteractiveSession()
            self.merge = tf.summary.merge_all()
            tf.global_variables_initializer().run()
            tf.local_variables_initializer().run()

            self.sess.graph.finalize()
            self.filewriter = tf.summary.FileWriter(logdir="./adda/logs/", graph=self.sess.graph)

        self.load_network()
        print("model init successfully!")
 def chi_sqr(z_true,z_pred):
     numerator = K.square(z_true - z_pred)
     denominator = tf.map_fn(lambda x: tf.where(tf.less(x, 1e-7), 1., x), z_true,  dtype='float32')        
     true_mean = tf.tensordot(z_true, distribution_elements_row, 1)
     hist_weights = 0.01 + K.square(distribution_elements_row - true_mean)        
     return tf.tensordot(tf.div(numerator,denominator), hist_weights, 1)
Esempio n. 50
0
def main():
    ################ check directory #########################################
    if not os.path.exists(a.output_dir):
        os.makedirs(a.output_dir)

    if a.checkpoint is None:
        raise Exception("checkpoint required for test mode")

    ############### display/save setting #####################################
    for k, v in a._get_kwargs():
        print(k, "=", v)

    with open(os.path.join(a.output_dir, "options.json"), "w") as f:
        f.write(json.dumps(vars(a), sort_keys=True, indent=4))

    ################ read TFRecord dataset ###################################
    input_batch, iterator = read_tfrecord()

    ################ creat model  ############################################
    # inputs and targets are of size [batch_size, height, width, channels]
    model = create_model(input_batch.images, input_batch.labels, a.class_num)

    ################ summaries ###############################################
    # undo colorization splitting on images that we use for display/output
    images = deprocess(input_batch.images)

    # YuhangLi: only save a part of images for saving driver space.
    with tf.name_scope("encode_images"):
        display_fetches = {
            #"paths": input_batch.paths,
            "images":
            tf.map_fn(tf.image.encode_png,
                      convert(images),
                      dtype=tf.string,
                      name="images"),
            "label":
            input_batch.labels
        }

    ################ configuration ##############################################
    with tf.name_scope("parameter_count"):
        parameter_count = tf.reduce_sum(
            [tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])

    saver = tf.train.Saver(max_to_keep=1)

    logdir = a.output_dir
    sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
    sess_config = tf.ConfigProto(allow_soft_placement=True)
    sess_config.gpu_options.allow_growth = True
    f = open("log.txt", mode='wb')

    ############### session ######################################################
    with sv.managed_session(config=sess_config) as sess:
        print("parameter_count =", sess.run(parameter_count))

        if a.checkpoint is not None:
            print("loading model from checkpoint")
            checkpoint = tf.train.latest_checkpoint(a.checkpoint)
            saver.restore(sess, checkpoint)

        max_steps = 2**32
        if a.max_epochs is not None:
            max_steps = input_batch.steps_per_epoch * a.max_epochs
        if a.max_steps is not None:
            max_steps = a.max_steps

        ######################## test ############################
        if a.mode == "test":
            # at most, process the test data once
            max_steps = min(input_batch.steps_per_epoch, max_steps)
            for step in range(max_steps):
                results = sess.run(display_fetches)
                filesets = save_images(results, step=step)
                for i, f in enumerate(filesets):
                    print("evaluated image", f["name"])
                # temporaly commented, error for unknown reason
                # index_path = append_index(filesets)

            print("wrote index at", index_path)
    return
Esempio n. 51
0
 def call(self, vector):
     """
     Parameters
         vector: tensor of affine components
     """
     return tf.map_fn(self._single_conversion, vector, dtype=tf.float32)
def _split_target(sequence_batch: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]:
    """Split a N + 1 sequence into shifted-by-1 sequences for input and output."""
    input_text = tf.map_fn(lambda x: x[:-1], sequence_batch)
    target_text = tf.map_fn(lambda x: x[1:], sequence_batch)
    return (input_text, target_text)
Esempio n. 53
0
            infer_decoder, output_time_major=True, swap_memory=True)
        preds = infer_decoder_outputs.sample_id

    return preds


int_to_vocab, vocab_size = load_vocab(FLAGS.vocab_file)

vocab = lookup_ops.index_table_from_file(FLAGS.vocab_file,
                                         default_value=FLAGS.unk_id)
eos_id = tf.cast(vocab.lookup(tf.constant(FLAGS.eos)), tf.int32)

raw_sequence_op = tf.placeholder(tf.string, [1, None])

sequence = tf.cast(vocab.lookup(raw_sequence_op), tf.int32)
sequence = tf.map_fn(lambda x: tf.concat((x, [eos_id]), 0), sequence)
source_sequence_length = tf.map_fn(lambda x: tf.size(x), sequence)

preds = create_network(sequence, FLAGS.sos, FLAGS.eos, vocab,
                       source_sequence_length, vocab_size, FLAGS.hidden_size,
                       FLAGS.batch_size, FLAGS.encoder_num_layers,
                       FLAGS.decoder_num_layers)

sess = tf.Session()

sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())

saver = tf.train.Saver()
latest_checkpoint = tf.train.latest_checkpoint('checkpoint_bahdanau')
if latest_checkpoint and tf.train.checkpoint_exists(latest_checkpoint):
Esempio n. 54
0
File: ard.py Progetto: mizunt1/ADVI
 def convert_alpha(self, alpha):
     one_over_sqrt_alpha = tf.map_fn(lambda x: 1 / (tf.math.sqrt(x)),
                                     alpha,
                                     dtype=tf.float64)
     return one_over_sqrt_alpha
Esempio n. 55
0
    def filter_predictions(batch_item):
        # Create a function that filters the predictions for one single class.
        def filter_single_class(index):
            # From a tensor of shape (n_boxes, n_classes + 4 coordinates) extract
            # a tensor of shape (n_boxes, 1 + 4 coordinates) that contains the
            # confidnece values for just one class, determined by `index`.
            confidences = tf.expand_dims(batch_item[..., index], axis=-1)
            # revised
            class_id = tf.fill(dims=tf.shape(confidences),
                               value=tf.cast(index, dtype=tf.float32))
            box_coordinates = batch_item[..., -4:]

            single_class = tf.concat([class_id, confidences, box_coordinates],
                                     axis=-1)

            # Apply confidence thresholding with respect to the class defined by `index`.
            threshold_met = single_class[:, 1] > tf_confidence_thresh
            single_class = tf.boolean_mask(tensor=single_class,
                                           mask=threshold_met)

            # If any boxes made the threshold, perform NMS.
            def perform_nms():
                scores = single_class[..., 1]

                # `tf.image.non_max_suppression()` needs the box coordinates in the format `(ymin, xmin, ymax, xmax)`.
                xmin = tf.expand_dims(single_class[..., -4], axis=-1)
                ymin = tf.expand_dims(single_class[..., -3], axis=-1)
                xmax = tf.expand_dims(single_class[..., -2], axis=-1)
                ymax = tf.expand_dims(single_class[..., -1], axis=-1)
                boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)

                maxima_indices = tf.image.non_max_suppression(
                    boxes=boxes,
                    scores=scores,
                    max_output_size=tf_nms_max_output_size,
                    iou_threshold=iou_threshold,
                    name='non_maximum_suppresion')
                maxima = tf.gather(params=single_class,
                                   indices=maxima_indices,
                                   axis=0)
                return maxima

            def no_confident_predictions():
                return tf.constant(value=0.0, shape=(1, 6), dtype=tf.float32)

            single_class_nms = tf.cond(tf.equal(tf.size(single_class), 0),
                                       no_confident_predictions, perform_nms)

            # Make sure `single_class` is exactly `self.nms_max_output_size` elements long.
            padded_single_class = tf.pad(tensor=single_class_nms,
                                         paddings=[[
                                             0, tf_nms_max_output_size -
                                             tf.shape(single_class_nms)[0]
                                         ], [0, 0]],
                                         mode='CONSTANT',
                                         constant_values=0.0)

            return padded_single_class

        # Iterate `filter_single_class()` over all class indices.
        filtered_single_classes = tf.map_fn(
            fn=lambda i: filter_single_class(i),
            elems=tf.range(1, n_classes),
            dtype=tf.float32,
            parallel_iterations=128,
            back_prop=False,
            swap_memory=False,
            infer_shape=True,
            name='loop_over_classes')

        # Concatenate the filtered results for all individual classes to one tensor.
        filtered_predictions = tf.reshape(tensor=filtered_single_classes,
                                          shape=(-1, 6))

        # Perform top-k filtering for this batch item or pad it in case there are
        # fewer than `self.top_k` boxes left at this point. Either way, produce a
        # tensor of length `self.top_k`. By the time we return the final results tensor
        # for the whole batch, all batch items must have the same number of predicted
        # boxes so that the tensor dimensions are homogenous. If fewer than `self.top_k`
        # predictions are left after the filtering process above, we pad the missing
        # predictions with zeros as dummy entries.
        def top_k():
            return tf.gather(params=filtered_predictions,
                             indices=tf.nn.top_k(filtered_predictions[:, 1],
                                                 k=tf_top_k,
                                                 sorted=True).indices,
                             axis=0)

        def pad_and_top_k():
            padded_predictions = tf.pad(
                tensor=filtered_predictions,
                paddings=[[0, tf_top_k - tf.shape(filtered_predictions)[0]],
                          [0, 0]],
                mode='CONSTANT',
                constant_values=0.0)
            return tf.gather(params=padded_predictions,
                             indices=tf.nn.top_k(padded_predictions[:, 1],
                                                 k=tf_top_k,
                                                 sorted=True).indices,
                             axis=0)

        top_k_boxes = tf.cond(
            tf.greater_equal(tf.shape(filtered_predictions)[0], tf_top_k),
            top_k, pad_and_top_k)

        return top_k_boxes
Esempio n. 56
0
def main():
    if dataset == 'cifar10':
        x_train, y_train = cifar_input.get_cifar_data(data_path, option='train')
        x_test, y_test = cifar_input.get_cifar_data(data_path, option='test')
    elif dataset == 'svhn':
        x_train, y_train, x_test, y_test = svhn_input.get_svhn_data('../datasets')

    model_x_train = []
    model_y_train = []

    sorted_x_train, sorted_y_train = noniid.generate_nonIID(x_train, y_train)
    temp_idx = [0] * num_classes

    cls_idx_ex = [] * 10
    cls_idx_ex.append([9, 7, 3, 6, 4, 8, 0, 5, 1, 2])
    cls_idx_ex.append([2, 0, 6, 7, 5, 8, 3, 4, 1, 9])
    cls_idx_ex.append([4, 3, 1, 0, 6, 9, 5, 8, 2, 7])
    cls_idx_ex.append([3, 0, 1, 4, 9, 5, 2, 8, 7, 6])
    cls_idx_ex.append([8, 4, 6, 1, 2, 9, 0, 5, 7, 3])
    cls_idx_ex.append([6, 9, 8, 4, 7, 2, 3, 5, 0, 1])
    cls_idx_ex.append([8, 6, 2, 4, 1, 7, 3, 0, 9, 5])
    cls_idx_ex.append([5, 1, 7, 6, 9, 3, 2, 0, 8, 4])
    cls_idx_ex.append([5, 6, 1, 3, 9, 0, 7, 2, 4, 8])
    cls_idx_ex.append([8, 0, 9, 1, 3, 4, 5, 7, 6, 2])

    temp_cls_idx = 0
    cls_idx = cls_idx_ex[temp_cls_idx]

    for i in range(num_learners):
        temp_x = np.concatenate([sorted_x_train[cls_idx[j]]
                                 [temp_idx[cls_idx[j]]:
                                  temp_idx[
                                      cls_idx[j]] + num_training_data_examples_per_learner / num_classes_per_learner]
                                 for j in range(num_classes_per_learner)])
        temp_y = np.concatenate([sorted_y_train[cls_idx[j]]
                                 [temp_idx[cls_idx[j]]:
                                  temp_idx[
                                      cls_idx[j]] + num_training_data_examples_per_learner / num_classes_per_learner]
                                 for j in range(num_classes_per_learner)])

        for j in range(num_classes_per_learner):
            temp_idx[cls_idx[j]] += num_training_data_examples_per_learner / num_classes_per_learner

        model_x_train.append(temp_x)
        model_y_train.append(temp_y)

        cls_idx = np.delete(cls_idx, np.s_[:num_classes_per_learner], axis=0)
        if len(cls_idx) == 0 and i < num_learners - 1:
            temp_cls_idx += 1
            cls_idx = cls_idx_ex[temp_cls_idx]

    del sorted_x_train
    del sorted_y_train

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = False
    config.allow_soft_placement = True

    model_batch_x = []
    model_batch_y = []

    model_pred = []
    model_cost = []

    model_schedule = []
    model_opt = []
    model_train_op = []

    model_accuracy = []

    model_params = []
    update_ops = []
    variable_set = []
    moving_variances = []
    moving_mean_squares = []
    moving_square_means = []
    moving_means = []

    model_is_train = []

    num_conv_chs = [3, 16 * num_convs_width_multi, 128, 256]
    num_fc_outdims = [num_classes, 256, 256 * num_fcs_width_multi]

    for i in range(num_learners + 2):
        with tf.variable_scope('model{}'.format(i)):
            with tf.name_scope('input'):
                model_batch_x.insert(i, tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x-input'))
                model_batch_y.insert(i, tf.placeholder(tf.float32, shape=[None, num_classes], name='y-input'))
                model_schedule.insert(i, tf.placeholder_with_default(0.0, shape=None))
                model_is_train.insert(i, tf.placeholder_with_default(False, shape=None, name="is_train"))

            if is_aug is True:
                out = tf.cond(model_is_train[i],
                              lambda: image_augmentation(model_batch_x[i]),
                              lambda: tf.map_fn(lambda img: tf.image.per_image_standardization(img), model_batch_x[i]))
            else:
                out = tf.map_fn(lambda img: tf.image.per_image_standardization(img), model_batch_x[i])

            with tf.variable_scope('conv_g1'):
                with tf.variable_scope('conv1'):
                    out = _conv(out, num_conv_chs[0], num_conv_chs[1], 1)
                    if is_bn is True: out = _batchnorm(out, model_is_train[i])
                    if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                out = tf.nn.relu(out)
                if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
                for j in six.moves.range(1, num_convs):
                    with tf.variable_scope('conv%d' % (j + 1)):
                        out = _conv(out, num_conv_chs[1], num_conv_chs[1], 1)
                        if is_bn is True: out = _batchnorm(out, model_is_train[i])
                        if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                    out = tf.nn.relu(out)
                    if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])

            out = tf.nn.max_pool(out, ksize=(1, 3, 3, 1), strides=[1, 2, 2, 1], padding='SAME')

            with tf.variable_scope('conv_g2'):
                with tf.variable_scope('conv1'):
                    out = _conv(out, num_conv_chs[1], num_conv_chs[2], 2)
                    if is_bn is True: out = _batchnorm(out, model_is_train[i])
                    if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                out = tf.nn.relu(out)
                if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
                for j in six.moves.range(1, num_convs):
                    with tf.variable_scope('conv%d' % (j + 1)):
                        out = _conv(out, num_conv_chs[2], num_conv_chs[2], 2)
                        if is_bn is True: out = _batchnorm(out, model_is_train[i])
                        if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                    out = tf.nn.relu(out)
                    if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])

            out = tf.nn.max_pool(out, ksize=(1, 3, 3, 1), strides=[1, 2, 2, 1], padding='SAME')

            with tf.variable_scope('conv_g3'):
                with tf.variable_scope('conv1'):
                    out = _conv(out, num_conv_chs[2], num_conv_chs[3], 3)
                    if is_bn is True: out = _batchnorm(out, model_is_train[i])
                    if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                out = tf.nn.relu(out)
                if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])
                for j in six.moves.range(1, num_convs):
                    with tf.variable_scope('conv%d' % (j + 1)):
                        out = _conv(out, num_conv_chs[3], num_conv_chs[3], 3)
                        if is_bn is True: out = _batchnorm(out, model_is_train[i])
                        if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                    out = tf.nn.relu(out)
                    if is_dropout is True: out = tf.layers.dropout(out, dropout_rate1, training=model_is_train[i])

            if last_pooling == 'max':
                out = tf.nn.max_pool(out, ksize=(1, 3, 3, 1), strides=[1, last_stride, last_stride, 1], padding='SAME')
            elif last_pooling == 'avg':
                out = _global_avg_pool(out)

            n = 1
            for j in range(1, len(out.get_shape().as_list())):
                n = n * out.get_shape().as_list()[j]
            out = tf.reshape(out, shape=[-1, n])

            for j in six.moves.range(1, num_fcs):
                temp_outdims = num_fc_outdims[2]
                if j == num_fcs - 1: temp_outdims = num_fc_outdims[1]
                with tf.variable_scope('fc%d' % (j)):
                    out = _fc(out, temp_outdims)
                    if is_bn is True: out = _batchnorm(out, model_is_train[i])
                    if is_brn is True: out = _batchrnorm(out, model_is_train[i])
                out = tf.nn.relu(out)
                if is_dropout is True: out = tf.layers.dropout(out, dropout_rate2, training=model_is_train[i])

            with tf.variable_scope('fc%d' % num_fcs):
                logits = _fc(out, num_fc_outdims[0])

            model_pred.insert(i, tf.nn.softmax(logits=logits))

            with tf.name_scope('cost'):
                xent1 = tf.nn.softmax_cross_entropy_with_logits(logits=logits,
                                                                labels=model_batch_y[i])
                model_cost.insert(i, tf.reduce_mean(xent1))

            lr = lr_init * model_schedule[i]
            wd = wd_init * model_schedule[i]

            with tf.name_scope('opt'):
                if opt_algo == 'pure':
                    model_opt.insert(i, tf.contrib.opt.MomentumWOptimizer(weight_decay=wd,
                                                                          learning_rate=lr,
                                                                          momentum=0.0))
                elif opt_algo == 'nmom-wb' or opt_algo == 'nmom-a':
                    model_opt.insert(i, tf.contrib.opt.MomentumWOptimizer(weight_decay=wd,
                                                                          learning_rate=lr,
                                                                          momentum=0.9,
                                                                          use_nesterov=True))
                elif opt_algo == 'adam-wb' or opt_algo == 'adam-a':
                    model_opt.insert(i, tf.contrib.opt.AdamWOptimizer(weight_decay=wd,
                                                                      learning_rate=lr))

                update_ops.insert(i, tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope='model{}'.format(i)))
                with tf.control_dependencies(update_ops[i]):
                    model_train_op.insert(i, model_opt[i].minimize(model_cost[i]))

            with tf.name_scope('Accuracy'):
                predictions = tf.argmax(model_pred[i], axis=1)
                model_accuracy.insert(i, tf.reduce_mean(
                    tf.to_float(tf.equal(predictions, tf.argmax(model_batch_y[i], axis=1)))))

        model_params.insert(i, tf.contrib.framework.get_trainable_variables(scope='model{}'.format(i)))
        variable_set.insert(i, tf.contrib.framework.get_variables(scope='model{}'.format(i),
                                                                  collection=tf.GraphKeys.VARIABLES))
        moving_means.insert(i, tf.contrib.framework.get_variables(scope='model{}'.format(i),
                                                                  suffix='moving_mean',
                                                                  collection=tf.GraphKeys.VARIABLES))
        moving_variances.insert(i, tf.contrib.framework.get_variables(scope='model{}'.format(i),
                                                                      suffix='moving_variance',
                                                                      collection=tf.GraphKeys.VARIABLES))

        moving_mean_squares.insert(i, [tf.math.square(v) for v in tf.contrib.framework.get_variables(scope='model{}'.format(i),
                                                                                                     suffix='moving_mean',
                                                                                                     collection=tf.GraphKeys.VARIABLES)])

        moving_square_means.insert(i, [var + msq  for var, msq in zip(moving_variances[i], moving_mean_squares[i])])

    init_params = []
    avg_assign_global = []
    avg_assign_wb = []
    avg_assign_a = []
    assign_var_params = []

    for i in range(num_learners + 2):
        init_params.insert(i, [])
        avg_assign_wb.insert(i, [])
        avg_assign_a.insert(i, [])
        for idx in range(len(model_params[0])):
            init_params[i].insert(idx, tf.assign(model_params[i][idx], model_params[0][idx]))

    variable_avg = []
    for idx in range(len(variable_set[0])):
        variable_avg.insert(idx, 0)
        for i in range(num_learners):
            variable_avg[idx] += variable_set[i][idx] / num_learners


    temp_idx = 0
    for idx in variable_set[0]:
        avg_assign_global.insert(temp_idx,
                                 tf.assign(variable_set[num_learners][temp_idx], variable_avg[temp_idx]))
        temp_idx += 1

    temp_idx = 0
    for idx in variable_set[0]:
        for i in range(num_learners + 2):
            if idx.op.name.find(r'MomentumW') == -1 and idx.op.name.find(r'AdamW') == -1:
                avg_assign_wb[i].insert(temp_idx, tf.assign(variable_set[i][temp_idx],
                                                            variable_set[num_learners][temp_idx]))
            else:
                avg_assign_wb[i].insert(temp_idx, tf.assign(variable_set[i][temp_idx],
                                                            variable_set[i][temp_idx]))
        temp_idx += 1

    temp_idx = 0
    for idx in variable_set[0]:
        for i in range(num_learners + 2):
            avg_assign_a[i].insert(temp_idx,
                                   tf.assign(variable_set[i][temp_idx],
                                             variable_set[num_learners][temp_idx]))
        temp_idx += 1

    moving_var_avg = []
    moving_mean_avg = []

    for idx in range(len(moving_means[0])):
        moving_mean_avg.insert(idx, 0)
        for i in range(num_learners):
            moving_mean_avg[idx] += moving_means[i][idx] / num_learners

    for idx in range(len(moving_variances[0])):
        moving_var_avg.insert(idx, 0)
        for i in range(num_learners):
            moving_var_avg[idx] += moving_square_means[i][idx] / num_learners
        moving_var_avg[idx] -= tf.math.square(moving_mean_avg[idx])

    temp_idx = 0
    for idx in moving_variances[0]:
        assign_var_params.insert(temp_idx,
                                 tf.assign(moving_variances[num_learners][temp_idx], moving_var_avg[temp_idx]))
        temp_idx += 1

    with tf.train.MonitoredTrainingSession(config=config) as sess:
        sess.run(init_params)
        training_loss = np.zeros([num_learners], dtype=np.float32)

        best_accuracy = 0.0
        best_accuracy_round = 0
        comm_round = 1
        while comm_round <= training_rounds:
            print('====================Round {0}===================='.format(comm_round))
            if num_learners == 1:
                if comm_round < 51:
                    schedule = 1.0
                elif comm_round >= 51 and comm_round < 76:
                    schedule = 0.1
                elif comm_round >= 76:
                    schedule = 0.01
            elif num_learners == 10 and num_classes_per_learner == 10:
                if comm_round < 101:
                    schedule = 1.0
                elif comm_round >= 101 and comm_round < 151:
                    schedule = 0.1
                elif comm_round >= 151:
                    schedule = 0.01
            elif num_learners == 10 and num_classes_per_learner == 2:
                if comm_round < 151:
                    schedule = 1.0
                elif comm_round >= 151 and comm_round < 226:
                    schedule = 0.1
                elif comm_round >= 226:
                    schedule = 0.01
            elif num_learners == 10 and num_classes_per_learner == 1:
                if comm_round < 151:
                    schedule = 1.0
                elif comm_round >= 151 and comm_round < 226:
                    schedule = 0.1
                elif comm_round >= 226:
                    schedule = 0.01

            training_loss.fill(0.0)

            shuffle_idx = [] * num_learners
            for i in range(num_learners):
                shuffle_idx.insert(i, np.arange(num_training_data_examples_per_learner, dtype=np.int32))
                np.random.shuffle(shuffle_idx[i])

            for j in range(local_steps):
                for i in range(num_learners):
                    batch_xs = model_x_train[i][shuffle_idx[i][:batch_size]]
                    batch_ys = model_y_train[i][shuffle_idx[i][:batch_size]]
                    shuffle_idx[i] = np.delete(shuffle_idx[i], np.s_[:batch_size], axis=0)
                    if len(shuffle_idx[i]) == 0:
                        shuffle_idx[i] = np.arange(num_training_data_examples_per_learner, dtype=np.int32)
                        np.random.shuffle(shuffle_idx[i])

                    _, loss = sess.run([model_train_op[i], model_cost[i]],
                                       feed_dict={model_batch_x[i]: batch_xs, model_batch_y[i]: batch_ys,
                                                  model_is_train[i]: True, model_schedule[i]: schedule})

                    training_loss[i] += loss / local_steps

            print('[1-1] Training Loss (Each Learner, Mean): {0}, {1}'.format(training_loss, np.mean(training_loss)))
            print(' ')

            if num_learners != 1:
                shuffle_idx = np.arange(num_training_data_examples, dtype=np.int32)
                np.random.shuffle(shuffle_idx)
                for idx in range(local_steps):
                    batch_xs = x_train[shuffle_idx[:batch_size]]
                    batch_ys = y_train[shuffle_idx[:batch_size]]
                    shuffle_idx = np.delete(shuffle_idx, np.s_[:batch_size], axis=0)
                    if len(shuffle_idx) == 0:
                        shuffle_idx = np.arange(num_training_data_examples)
                        np.random.shuffle(shuffle_idx)

                    sess.run(model_train_op[num_learners + 1],
                             feed_dict={model_batch_x[num_learners + 1]: batch_xs,
                                        model_batch_y[num_learners + 1]: batch_ys,
                                        model_is_train[num_learners + 1]: True,
                                        model_schedule[num_learners + 1]: schedule})

                model_params_per_layer = [] * (num_learners + 2)
                model_params_whole = [] * (num_learners + 2)
                for i in range(num_learners + 2):
                    temp_params = sess.run(model_params[i])

                    temp_params_per_layer = []
                    temp_params_whole = np.array((), dtype=float)

                    temp_idx = 0
                    for idx in model_params[0]:
                        if idx.op.name.find(r'weight') > 0:
                            temp_params_per_layer.append(temp_params[temp_idx].flatten())
                            temp_params_whole = np.append(temp_params_whole, temp_params[temp_idx].flatten())
                        temp_idx += 1

                    model_params_per_layer.insert(i, temp_params_per_layer)
                    model_params_whole.insert(i, temp_params_whole)

                dist_per_layer = [0.0] * len(model_params_per_layer[0])
                for i in range(num_learners):
                    for j in range(num_learners):
                        if i != j:
                            for idx in range(len(model_params_per_layer[0])):
                                dist_per_layer[idx] += distance.cosine(
                                    model_params_per_layer[i][idx], model_params_per_layer[j][idx]
                                ) / (num_learners * num_learners - num_learners)

                print('[2-1] PD_Ls (Each Layer): {0}'.format(dist_per_layer))

                dist_per_layer = [0.0] * len(model_params_per_layer[0])

                for i in range(num_learners):
                    for idx in range(len(model_params_per_layer[0])):
                        dist_per_layer[idx] += distance.cosine(
                            model_params_per_layer[i][idx], model_params_per_layer[num_learners + 1][idx]
                        ) / num_learners

                print('[2-2] PD_VL (Each Layer): {0}'.format(dist_per_layer))

            sess.run(avg_assign_global)
            sess.run(assign_var_params)

            test_accuracy = 0.0
            test_iters = 100
            test_batch_size = num_test_data_examples / test_iters
            for j in range(test_iters):
                temp_accuracy = sess.run(model_accuracy[num_learners],
                                         feed_dict={model_batch_x[num_learners]:
                                                        x_test[test_batch_size * j:test_batch_size * (j + 1)],
                                                    model_batch_y[num_learners]:
                                                        y_test[test_batch_size * j:test_batch_size * (j + 1)],
                                                    model_is_train[num_learners]: False})
                test_accuracy += temp_accuracy
            test_accuracy = test_accuracy / float(test_iters)

            if test_accuracy > best_accuracy:
                best_accuracy = test_accuracy
                best_accuracy_round = comm_round

            print('[3-1] Test Accuracy (Global Model) - Current Round : {0}'.format(test_accuracy))
            print('[3-2] Test Accuracy (Global Model) - Best : {0}'.format(best_accuracy))
            print('[3-3] Round - Best Test Accuracy (Global Model) : {0}'.format(best_accuracy_round))
            print(' ')

            if opt_algo == 'pure' or opt_algo == 'nmom-wb' or opt_algo == 'adam-wb':
                for i in range(num_learners):
                    sess.run(avg_assign_wb[i])
                sess.run(avg_assign_wb[num_learners + 1])
            elif opt_algo == 'nmom-a' or opt_algo == 'adam-a':
                for i in range(num_learners):
                    sess.run(avg_assign_a[i])
                sess.run(avg_assign_a[num_learners + 1])

            comm_round += 1



    print('Done')
    print('Session closed cleanly')
Esempio n. 57
0
def DecodeDetections(
    y_pred,
    confidence_thresh=0.01,
    iou_threshold=0.45,
    top_k=200,
    nms_max_output_size=400,
    coords='centroids',
    normalize_coords=True,
    img_height=None,
    img_width=None,
):
    '''
    All default argument values follow the Caffe implementation.
    Arguments:
        confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
            positive class in order to be considered for the non-maximum suppression stage for the respective class.
            A lower value will result in a larger part of the selection process being done by the non-maximum suppression
            stage, while a larger value will result in a larger part of the selection process happening in the confidence
            thresholding stage.
        iou_threshold (float, optional): A float in [0,1]. All boxes with a Jaccard similarity of greater than `iou_threshold`
            with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
            to the box score.
        top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
            non-maximum suppression stage.
        nms_max_output_size (int, optional): The maximum number of predictions that will be left after performing non-maximum
            suppression.
        coords (str, optional): The box coordinate format that the model outputs. Must be 'centroids'
            i.e. the format `(cx, cy, w, h)` (box center coordinates, width, and height). Other coordinate formats are
            currently not supported.
        normalize_coords (bool, optional): Set to `True` if the model outputs relative coordinates (i.e. coordinates in [0,1])
            and you wish to transform these relative coordinates back to absolute coordinates. If the model outputs
            relative coordinates, but you do not want to convert them back to absolute coordinates, set this to `False`.
            Do not set this to `True` if the model already outputs absolute coordinates, as that would result in incorrect
            coordinates. Requires `img_height` and `img_width` if set to `True`.
        img_height (int, optional): The height of the input images. Only needed if `normalize_coords` is `True`.
        img_width (int, optional): The width of the input images. Only needed if `normalize_coords` is `True`.
    Returns:
            3D tensor of shape `(batch_size, top_k, 6)`. The second axis is zero-padded
            to always yield `top_k` predictions per batch item. The last axis contains
            the coordinates for each predicted box in the format
            `[class_id, confidence, xmin, ymin, xmax, ymax]`.
    '''
    # We need these members for TensorFlow.
    tf_confidence_thresh = tf.constant(confidence_thresh,
                                       dtype=tf.float32,
                                       name='confidence_thresh')
    tf_iou_threshold = tf.constant(iou_threshold,
                                   dtype=tf.float32,
                                   name='iou_threshold')
    tf_top_k = tf.constant(top_k, name='top_k')
    tf_normalize_coords = tf.constant(normalize_coords,
                                      name='normalize_coords')
    tf_img_height = tf.constant(img_height,
                                dtype=tf.float32,
                                name='img_height')
    tf_img_width = tf.constant(img_width, dtype=tf.float32, name='img_width')
    tf_nms_max_output_size = tf.constant(nms_max_output_size,
                                         name='nms_max_output_size')

    #####################################################################################
    # 1. Convert the box coordinates from predicted anchor box offsets to predicted
    #    absolute coordinates
    #####################################################################################

    # Convert anchor box offsets to image offsets.
    cx = y_pred[..., -12] * y_pred[..., -4] * y_pred[..., -6] + y_pred[
        ..., -8]  # cx = cx_pred * cx_variance * w_anchor + cx_anchor
    cy = y_pred[..., -11] * y_pred[..., -3] * y_pred[..., -5] + y_pred[
        ..., -7]  # cy = cy_pred * cy_variance * h_anchor + cy_anchor
    w = tf.exp(y_pred[..., -10] * y_pred[..., -2]) * y_pred[
        ..., -6]  # w = exp(w_pred * variance_w) * w_anchor
    h = tf.exp(y_pred[..., -9] * y_pred[..., -1]) * y_pred[
        ..., -5]  # h = exp(h_pred * variance_h) * h_anchor

    # Convert 'centroids' to 'corners'.
    xmin = cx - 0.5 * w
    ymin = cy - 0.5 * h
    xmax = cx + 0.5 * w
    ymax = cy + 0.5 * h

    # If the model predicts box coordinates relative to the image dimensions and they are supposed
    # to be converted back to absolute coordinates, do that.
    def normalized_coords():
        xmin1 = tf.expand_dims(xmin * tf_img_width, axis=-1)
        ymin1 = tf.expand_dims(ymin * tf_img_height, axis=-1)
        xmax1 = tf.expand_dims(xmax * tf_img_width, axis=-1)
        ymax1 = tf.expand_dims(ymax * tf_img_height, axis=-1)
        return xmin1, ymin1, xmax1, ymax1

    def non_normalized_coords():
        return tf.expand_dims(xmin, axis=-1), tf.expand_dims(ymin, axis=-1), \
               tf.expand_dims(xmax, axis=-1), tf.expand_dims(ymax, axis=-1)

    xmin, ymin, xmax, ymax = tf.cond(tf_normalize_coords, normalized_coords,
                                     non_normalized_coords)

    # Concatenate the one-hot class confidences and the converted box coordinates to form the decoded predictions tensor.
    y_pred = tf.concat(values=[y_pred[..., :-12], xmin, ymin, xmax, ymax],
                       axis=-1)

    #####################################################################################
    # 2. Perform confidence thresholding, per-class non-maximum suppression, and
    #    top-k filtering.
    #####################################################################################
    batch_size = tf.shape(y_pred)[0]  # Output dtype: tf.int32
    n_boxes = tf.shape(y_pred)[1]
    n_classes = y_pred.shape[2] - 4
    class_indices = tf.range(1, n_classes)

    # Create a function that filters the predictions for the given batch item. Specifically, it performs:
    # - confidence thresholding
    # - non-maximum suppression (NMS)
    # - top-k filtering
    def filter_predictions(batch_item):
        # Create a function that filters the predictions for one single class.
        def filter_single_class(index):
            # From a tensor of shape (n_boxes, n_classes + 4 coordinates) extract
            # a tensor of shape (n_boxes, 1 + 4 coordinates) that contains the
            # confidnece values for just one class, determined by `index`.
            confidences = tf.expand_dims(batch_item[..., index], axis=-1)
            # revised
            class_id = tf.fill(dims=tf.shape(confidences),
                               value=tf.cast(index, dtype=tf.float32))
            box_coordinates = batch_item[..., -4:]

            single_class = tf.concat([class_id, confidences, box_coordinates],
                                     axis=-1)

            # Apply confidence thresholding with respect to the class defined by `index`.
            threshold_met = single_class[:, 1] > tf_confidence_thresh
            single_class = tf.boolean_mask(tensor=single_class,
                                           mask=threshold_met)

            # If any boxes made the threshold, perform NMS.
            def perform_nms():
                scores = single_class[..., 1]

                # `tf.image.non_max_suppression()` needs the box coordinates in the format `(ymin, xmin, ymax, xmax)`.
                xmin = tf.expand_dims(single_class[..., -4], axis=-1)
                ymin = tf.expand_dims(single_class[..., -3], axis=-1)
                xmax = tf.expand_dims(single_class[..., -2], axis=-1)
                ymax = tf.expand_dims(single_class[..., -1], axis=-1)
                boxes = tf.concat(values=[ymin, xmin, ymax, xmax], axis=-1)

                maxima_indices = tf.image.non_max_suppression(
                    boxes=boxes,
                    scores=scores,
                    max_output_size=tf_nms_max_output_size,
                    iou_threshold=iou_threshold,
                    name='non_maximum_suppresion')
                maxima = tf.gather(params=single_class,
                                   indices=maxima_indices,
                                   axis=0)
                return maxima

            def no_confident_predictions():
                return tf.constant(value=0.0, shape=(1, 6), dtype=tf.float32)

            single_class_nms = tf.cond(tf.equal(tf.size(single_class), 0),
                                       no_confident_predictions, perform_nms)

            # Make sure `single_class` is exactly `self.nms_max_output_size` elements long.
            padded_single_class = tf.pad(tensor=single_class_nms,
                                         paddings=[[
                                             0, tf_nms_max_output_size -
                                             tf.shape(single_class_nms)[0]
                                         ], [0, 0]],
                                         mode='CONSTANT',
                                         constant_values=0.0)

            return padded_single_class

        # Iterate `filter_single_class()` over all class indices.
        filtered_single_classes = tf.map_fn(
            fn=lambda i: filter_single_class(i),
            elems=tf.range(1, n_classes),
            dtype=tf.float32,
            parallel_iterations=128,
            back_prop=False,
            swap_memory=False,
            infer_shape=True,
            name='loop_over_classes')

        # Concatenate the filtered results for all individual classes to one tensor.
        filtered_predictions = tf.reshape(tensor=filtered_single_classes,
                                          shape=(-1, 6))

        # Perform top-k filtering for this batch item or pad it in case there are
        # fewer than `self.top_k` boxes left at this point. Either way, produce a
        # tensor of length `self.top_k`. By the time we return the final results tensor
        # for the whole batch, all batch items must have the same number of predicted
        # boxes so that the tensor dimensions are homogenous. If fewer than `self.top_k`
        # predictions are left after the filtering process above, we pad the missing
        # predictions with zeros as dummy entries.
        def top_k():
            return tf.gather(params=filtered_predictions,
                             indices=tf.nn.top_k(filtered_predictions[:, 1],
                                                 k=tf_top_k,
                                                 sorted=True).indices,
                             axis=0)

        def pad_and_top_k():
            padded_predictions = tf.pad(
                tensor=filtered_predictions,
                paddings=[[0, tf_top_k - tf.shape(filtered_predictions)[0]],
                          [0, 0]],
                mode='CONSTANT',
                constant_values=0.0)
            return tf.gather(params=padded_predictions,
                             indices=tf.nn.top_k(padded_predictions[:, 1],
                                                 k=tf_top_k,
                                                 sorted=True).indices,
                             axis=0)

        top_k_boxes = tf.cond(
            tf.greater_equal(tf.shape(filtered_predictions)[0], tf_top_k),
            top_k, pad_and_top_k)

        return top_k_boxes

    # Iterate `filter_predictions()` over all batch items.
    output_tensor = tf.map_fn(fn=lambda x: filter_predictions(x),
                              elems=y_pred,
                              dtype=None,
                              parallel_iterations=128,
                              back_prop=False,
                              swap_memory=False,
                              infer_shape=True,
                              name='loop_over_batch')

    return output_tensor
Esempio n. 58
0
def build_graph(train):
    with tf.device("/cpu:0"):
        with tf.name_scope('X'):
            x = tf.placeholder(tf.float32, [None, 125, 160, 1], name='x')
            mlp = x

        if train:
            with tf.name_scope('RANDOM-CROP-FLIP'):
                crop_x = tf.map_fn(lambda img: tf.random_crop(img, [112, 144, 1]), mlp)
                crop_x = tf.map_fn(lambda img: tf.image.random_flip_up_down(img), crop_x)
                mlp = crop_x
        else:
            with tf.name_scope('CENTER-CROP'):
                crop_x = tf.map_fn(lambda img: tf.image.resize_image_with_crop_or_pad(img, 112, 144), mlp)
                mlp = crop_x

    with tf.name_scope('CONV-1'):
        c1 = 16
        res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c1-1]])
        mlp = conv(mlp, weight([3, 3, 1, c1], name='w11')) + positive_bias([c1], name='b11')
        mlp = tf.nn.relu(mlp, name='conv1')
        mlp = conv(mlp, weight([3, 3, c1, c1], name='w12')) + positive_bias([c1], name='b12')
        mlp = tf.nn.relu(mlp, name='conv2')
        mlp = conv(mlp, weight([3, 3, c1, c1], name='w13')) + positive_bias([c1], name='b13')
        mlp = tf.nn.relu(mlp, name='conv3')
        # mlp = conv(mlp, weight([3, 3, c1, c1], name='w14')) + positive_bias([c1], name='b14')
        # mlp = tf.nn.relu(mlp, name='conv4')
        mlp = tf.add(mlp, res, name='res')
        mlp = pool(mlp, name='pool')

    with tf.name_scope('CONV-2'):
        c2 = 32
        res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c2-c1]])
        mlp = conv(mlp, weight([3, 3, c1, c2], name='w21')) + positive_bias([c2], name='b21')
        mlp = tf.nn.relu(mlp, name='conv1')
        mlp = conv(mlp, weight([3, 3, c2, c2], name='w22')) + positive_bias([c2], name='b22')
        mlp = tf.nn.relu(mlp, name='conv2')
        mlp = conv(mlp, weight([3, 3, c2, c2], name='w23')) + positive_bias([c2], name='b23')
        mlp = tf.nn.relu(mlp, name='conv3')
        # mlp = conv(mlp, weight([3, 3, c2, c2], name='w24')) + positive_bias([c2], name='b24')
        # mlp = tf.nn.relu(mlp, name='conv4')
        mlp = tf.add(mlp, res, name='res')
        mlp = pool(mlp, name='pool')

    with tf.name_scope('CONV-3'):
        c3 = 64
        res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c3-c2]])
        mlp = conv(mlp, weight([3, 3, c2, c3], name='w31')) + positive_bias([c3], name='b31')
        mlp = tf.nn.relu(mlp, name='conv1')
        mlp = conv(mlp, weight([3, 3, c3, c3], name='w32')) + positive_bias([c3], name='b32')
        mlp = tf.nn.relu(mlp, name='conv2')
        # mlp = conv(mlp, weight([3, 3, c3, c3], name='w33')) + positive_bias([c3], name='b33')
        # mlp = tf.nn.relu(mlp, name='conv3')
        # mlp = conv(mlp, weight([3, 3, c3, c3], name='w34')) + positive_bias([c3], name='b34')
        # mlp = tf.nn.relu(mlp, name='conv4')
        mlp = tf.add(mlp, res, name='res')
        mlp = pool(mlp, name='pool')

    with tf.name_scope('CONV-4'):
        c4 = 128
        res = tf.pad(mlp, [[0, 0], [0, 0], [0, 0], [0, c4-c3]])
        mlp = conv(mlp, weight([3, 3, c3, c4], name='w41')) + positive_bias([c4], name='b41')
        mlp = tf.nn.relu(mlp, name='conv1')
        mlp = conv(mlp, weight([3, 3, c4, c4], name='w42')) + positive_bias([c4], name='b42')
        mlp = tf.nn.relu(mlp, name='conv2')
        # mlp = conv(mlp, weight([3, 3, c4, c4], name='w43')) + positive_bias([c4], name='b43')
        # mlp = tf.nn.relu(mlp, name='conv3')
        # mlp = conv(mlp, weight([3, 3, c4, c4], name='w44')) + positive_bias([c4], name='b44')
        # mlp = tf.nn.relu(mlp, name='conv4')
        mlp = tf.add(mlp, res, name='res')
        mlp = pool(mlp, name='pool')

    with tf.name_scope('MASK'):
        ca = 66
        mask = tf.reshape(mlp, [-1, c4])
        mask = tf.nn.xw_plus_b(mask, weight([c4, ca], 'w5'), zero_bias([ca], 'b5'))
        mask = tf.tanh(mask)
        mask = tf.nn.xw_plus_b(mask, weight([ca, 1], 'w6'), zero_bias([1], 'b6'))
        mask = tf.reshape(mask, [-1, 7*9])
        mask = tf.nn.softmax(mask)
        mask = tf.reshape(mask, [-1, 7, 9, 1])

        mlp = tf.mul(mlp, mask)
        mlp = tf.reduce_sum(mlp, [1, 2], True)

    if train:
        with tf.name_scope('DROPOUT'):
            mlp = tf.nn.dropout(mlp, 0.5, noise_shape=tf.shape(mlp)*[1, 0, 0, 1]+[0, 1, 1, 0], name='dropout')  # dropout by map

    with tf.name_scope('FLAT'):
        mlp = tf.reshape(mlp, [-1, c4], name='flat')

    '''
    if train:
        with tf.name_scope('DROPOUT'):
            mlp = tf.nn.dropout(mlp, 0.5, name='dropout')
    '''

    # 1FC
    with tf.name_scope('FC'):
        mlp = tf.nn.xw_plus_b(mlp, weight([c4, 7], name='w7'), zero_bias([7], name='b7'), name='fc')

    with tf.name_scope('Y'):
        y = tf.placeholder(tf.float32, [None, 7], name='y')

    with tf.name_scope('SOFTMAX-WITH-LOSS'):
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y), name='loss')

    with tf.name_scope('SOFTMAX'):
        prob = tf.nn.softmax(mlp, name='prob')

    with tf.name_scope('ACC'):
        acc = tf.equal(tf.argmax(prob, 1), tf.argmax(y, 1), name='correct')
        acc = tf.reduce_mean(tf.cast(acc, tf.float32), name='acc')

    if train:
        with tf.name_scope('OPT'):
            opt = tf.train.AdamOptimizer(name='opt')
            train_op = opt.minimize(loss, name='train_op')
    else:
        train_op = None

    # 创建summary
    with tf.name_scope('SUM'):
        # mask 为方便展示仅尺度变化
        # mask_m = tf.reduce_min(mask, [1, 2], True)
        mask_M = tf.reduce_max(mask, [1, 2], True)
        mask_visual = mask / mask_M * 255.0  # mask_visual = (mask-mask_m) / (mask_M-mask_m) * 255.0

        # prj_mask 为方便展示仅尺度变化
        prj_mask = mask
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.nn.conv2d(prj_mask, tf.ones([3, 3, 1, 1])/9.0, strides=[1, 1, 1, 1], padding='SAME')
        prj_mask = tf.image.resize_nearest_neighbor(prj_mask, tf.shape(prj_mask)[1:3]*2)
        # prj_mask_m = tf.reduce_min(prj_mask, [1, 2], True)
        prj_mask_M = tf.reduce_max(prj_mask, [1, 2], True)
        prj_mask_visual = prj_mask / prj_mask_M * 255.0   # prj_mask_visual = (prj_mask-prj_mask_m) / (prj_mask_M-prj_mask_m) * 255.0

        # mask_crop_x 为方便展示动态范围变化
        mask_crop_x = prj_mask * crop_x
        mask_crop_x_m = tf.reduce_min(mask_crop_x, [1, 2], True)
        mask_crop_x_M = tf.reduce_max(mask_crop_x, [1, 2], True)
        mask_crop_x_visual = (mask_crop_x - mask_crop_x_m) / (mask_crop_x_M - mask_crop_x_m) * 255.0

        # y
        y_visual = tf.reshape(y, [-1, 1, 7, 1]) * 255.0

        # prob
        prob_visual = tf.reshape(prob, [-1, 1, 7, 1]) * 255.0

    if train:
        summary = tf.merge_summary([
                tf.image_summary('train mask', mask_visual),  # 1 7 9 1
                tf.image_summary('train prj_mask', prj_mask_visual),  # 1 112 144 1
                tf.image_summary('train crop_x', crop_x),  # 1 112 144 1
                tf.image_summary('train mask_crop_x', mask_crop_x_visual),  # 1 112 144 1
                tf.image_summary('train y', y_visual),  # 1 1 7 1
                tf.image_summary('train prob', prob_visual),  # 1 1 7 1
                tf.scalar_summary('train loss', loss),
                tf.scalar_summary('train acc', acc),
            ])
    else:
        summary = tf.merge_summary([
                tf.image_summary('val mask', mask_visual),  # 1 7 9 1
                tf.image_summary('val prj_mask', prj_mask_visual),  # 1 112 144 1
                tf.image_summary('val crop_x', crop_x),  # 1 112 144 1
                tf.image_summary('val mask_crop_x', mask_crop_x_visual),  # 1 112 144 1
                tf.image_summary('val y', y_visual),  # 1 1 7 1
                tf.image_summary('val prob', prob_visual),  # 1 1 7 1
                tf.scalar_summary('val loss', loss),
                tf.scalar_summary('val acc', acc),
            ])

    return [x, y, loss, prob, acc, train_op, summary, crop_x, mask]
    def call(self, inputs):
        if self.r_num == 1:
            outputs = K.dot(
                K.reshape(inputs, (-1, self.ch_i * self.n_i)),
                K.reshape(self.w,
                          (self.ch_i * self.n_i, self.ch_j * self.n_j)))
            outputs = squeeze(K.reshape(outputs, (-1, self.ch_j, self.n_j)))
        else:
            wr = K.reshape(self.w, (self.ch_i, self.n_i, self.ch_j * self.n_j))

            u = tf.transpose(tf.matmul(tf.transpose(inputs, [1, 0, 2]), wr),
                             [1, 0, 2])

            u = K.reshape(u, (-1, self.ch_i, self.ch_j, self.n_j))

            def rt(ub):
                ub = K.reshape(ub, (-1, self.ch_i, self.ch_j, self.n_j))
                ub_wo_g = K.stop_gradient(ub)
                b = 0.0
                for r in range(self.r_num):
                    if r > 0:
                        c = K.expand_dims(
                            K.softmax(b * self.b_alphas[r])
                        ) * self.ch_j  # distribution of weighs of capsules in I across capsules in J
                        c = K.stop_gradient(c)
                    else:
                        c = 1.0

                    if r == self.r_num - 1:
                        cub = c * ub
                    else:
                        cub = c * ub_wo_g
                    s = K.sum(cub, axis=-3)  # vectors of capsules in J
                    v = squeeze(s)  # squeezed vectors of capsules in J
                    if r == self.r_num - 1:
                        break

                    v = K.stop_gradient(v)

                    a = tf.einsum('bjk,bijk->bij', v, ub)  # a = v dot u
                    #a = K.matmul(K.reshape(v, (-1, 1, J, 1, n_j)),
                    #             K.reshape(u, (-1, I, J, n_j, 1))).reshape((-1, I, J))

                    b = b + a  # increase those b[i,j] where v[j] dot b[i,j] is larger
                return v

            u = K.reshape(u, (-1, self.ch_i * self.ch_j * self.n_j))

            global useGPU

            if useGPU:
                outputs = rt(u)
            else:
                outputs = tf.map_fn(rt,
                                    u,
                                    parallel_iterations=100,
                                    back_prop=True,
                                    infer_shape=False)

            outputs = K.reshape(outputs, (-1, self.ch_j, self.n_j))

        return outputs
Esempio n. 60
0
    def build_detection(self):
        self.embeds, self.embeds2 = self.get_image_embedding(
            self.search_images, reuse=True)
        alpha = 0.5  # 0.5
        beta = 0.2  # 0.3

        with tf.variable_scope('detection'):

            def _translation_match(
                    x, z):  # translation match for one example within a batch
                x = tf.expand_dims(x,
                                   0)  # [1, in_height, in_width, in_channels]
                z = tf.expand_dims(
                    z, -1)  # [filter_height, filter_width, in_channels, 1]
                y = tf.nn.conv2d(x,
                                 z,
                                 strides=[1, 1, 1, 1],
                                 padding='VALID',
                                 name='translation_match')
                return y

            #def _dep_translation_match(x, z):  # translation match for one example within a batch
            #x = tf.expand_dims(x, 0)  # [1, in_height, in_width, in_channels]
            #z = tf.expand_dims(z, -1)  # [filter_height, filter_width, in_channels, 1]
            #y = tf.nn.depthwise_conv2d(x, z, strides=[1, 1, 1, 1], padding='VALID', name='dep_translation_match')
            #return y

            #@functools.wraps(attention_module)
            #def att_fn(images, reuse=False):
            #with slim.arg_scope(self.arg_scope):
            #y, _ = attention_module(images, reuse=reuse)
            #return y

            with tf.variable_scope('patch_matching'):
                template = tf.add_n(self.pseu_temp) / (len(self.pseu_temp))
                template = alpha * self.templates + (1. - alpha) * template

                embeds = tf.split(self.embeds, 2, axis=-1)
                template = tf.split(template, 2, axis=-1)
                #embeds = self.embeds

                #output = tf.map_fn(lambda x: _translation_match(x, template[0]),
                #embeds,
                #dtype=embeds.dtype)
                #output = tf.squeeze(output, [1])  # of shape e.g., [b, h, w, n]

                #output = tf.map_fn(lambda x: _dep_translation_match(x, template[0]),
                #embeds,
                #dtype=embeds.dtype)
                #output = tf.squeeze(output, [1])  # of shape e.g., [b, h, w, n]
                #output = att_fn(output)

                output_s1 = tf.map_fn(
                    lambda x: _translation_match(x, template[0][0]),
                    embeds[0],
                    dtype=self.embeds.dtype)
                output_s1 = tf.squeeze(output_s1,
                                       [1])  # of shape e.g., [b, h, w, 1]

                output_s2 = tf.map_fn(
                    lambda x: _translation_match(x, template[1][0]),
                    embeds[1],
                    dtype=self.embeds.dtype)
                output_s2 = tf.squeeze(output_s2,
                                       [1])  # of shape e.g., [b, h, w, 1]

            with tf.variable_scope('patch_matching2'):
                template2 = tf.add_n(self.pseu_temp2) / (len(self.pseu_temp2))
                template2 = alpha * self.templates2 + (1. - alpha) * template2

                embeds2 = tf.split(self.embeds2, 2, axis=-1)
                template2 = tf.split(template2, 2, axis=-1)
                #embeds2 = self.embeds2

                #output2 = tf.map_fn(lambda x: _translation_match(x, template2[0]),
                #embeds2,
                #dtype=embeds2.dtype)
                #output2 = tf.squeeze(output2, [1])  # of shape e.g., [b, h, w, n]

                #output2 = tf.map_fn(lambda x: _dep_translation_match(x, template2[0]),
                #embeds2,
                #dtype=embeds2.dtype)
                #output2 = tf.squeeze(output2, [1])  # of shape e.g., [b, h, w, n]
                #output2 = att_fn(output2)

                output2_s1 = tf.map_fn(
                    lambda x: _translation_match(x, template2[0][0]),
                    embeds2[0],
                    dtype=self.embeds2.dtype)
                output2_s1 = tf.squeeze(output2_s1,
                                        [1])  # of shape e.g., [b, h, w, 1]

                output2_s2 = tf.map_fn(
                    lambda x: _translation_match(x, template2[1][0]),
                    embeds2[1],
                    dtype=self.embeds2.dtype)
                output2_s2 = tf.squeeze(output2_s2,
                                        [1])  # of shape e.g., [b, h, w, 1]

            with slim.arg_scope(self.arg_scope):
                #output = slim.batch_norm(output, scope='bn_patch')
                #output2 = slim.batch_norm(output2, scope='bn_patch2')

                output_s1 = slim.batch_norm(output_s1, scope='bn_patch_s1')
                output_s2 = slim.batch_norm(output_s2, scope='bn_patch_s2')
                output2_s1 = slim.batch_norm(output2_s1, scope='bn_patch2_s1')
                output2_s2 = slim.batch_norm(output2_s2, scope='bn_patch2_s2')

                #output = slim.batch_norm(output, scope='bn_patch_ext')
                #output2 = slim.batch_norm(output2, scope='bn_patch2_ext')

            output = output_s1 * beta + output_s2 * (1. - beta)
            output2 = output2_s1 * beta + output2_s2 * (1. - beta)
            outputF_all = tf.squeeze(output * 0.5 + output2 * 0.5, -1)
            self.response = outputF_all