Ejemplo n.º 1
0
def dot(a, b):
    with tf.op_scope([a, b], 'dot'):
        # TODO: implement N-dimensinal dot product that consistent with Numpy.
        a_shape = a.get_shape().as_list()
        a_dims = len(a_shape)
        b_shape = b.get_shape().as_list()
        b_dims = len(b_shape)

        # scalar dot scalar, scalar dot tensor or tensor dot scalar: just do element-wise multiply.
        if a_dims == 0 or b_dims == 0:
            return a * b

        # vector dot vector, where we can just perform element-wise prod, and then sum them all.
        if a_dims == 1 and b_dims == 1:
            return tf.reduce_sum(a * b)

        # vector dot matrix or matrix dot vector, where we should expand the vector to matrix, and then squeeze result.
        if a_dims <= 2 and b_dims <= 2:
            if a_dims == 1:
                a = tf.expand_dims(a, dim=0)
            if b_dims == 1:
                b = tf.expand_dims(b, dim=1)
            ret = tf.matmul(a, b)
            if a_dims == 1:
                ret = tf.squeeze(ret, [0])
            if b_dims == 1:
                ret = tf.squeeze(ret, [1])
            return ret

    # throw exception, that we do not know how to handle the situation.
    raise TypeError('Tensor dot between shape %r and %r is not supported.' % (a_shape, b_shape))
Ejemplo n.º 2
0
def from_logits(behaviour_policy_logits,
                target_policy_logits,
                actions,
                discounts,
                rewards,
                values,
                bootstrap_value,
                clip_rho_threshold=1.0,
                clip_pg_rho_threshold=1.0,
                name='vtrace_from_logits'):
    """multi_from_logits wrapper used only for tests"""

    res = multi_from_logits(
        [behaviour_policy_logits], [target_policy_logits], [actions],
        discounts,
        rewards,
        values,
        bootstrap_value,
        clip_rho_threshold=clip_rho_threshold,
        clip_pg_rho_threshold=clip_pg_rho_threshold,
        name=name)

    return VTraceFromLogitsReturns(
        vs=res.vs,
        pg_advantages=res.pg_advantages,
        log_rhos=res.log_rhos,
        behaviour_action_log_probs=tf.squeeze(
            res.behaviour_action_log_probs, axis=0),
        target_action_log_probs=tf.squeeze(
            res.target_action_log_probs, axis=0),
    )
    def get_reconstructed_image(self, real, imag, name=None):
        """
        :param real:
        :param imag:
        :param name:
        :return:
        """
        complex_k_space_label = tf.complex(real=tf.squeeze(real), imag=tf.squeeze(imag), name=name+"_complex_k_space")
        rec_image_complex = tf.expand_dims(tf.ifft2d(complex_k_space_label), axis=1)
        
        rec_image_real = tf.reshape(tf.real(rec_image_complex), shape=[-1, 1, self.dims_out[1], self.dims_out[2]])
        rec_image_imag = tf.reshape(tf.imag(rec_image_complex), shape=[-1, 1, self.dims_out[1], self.dims_out[2]])

        # Shifting
        top, bottom = tf.split(rec_image_real, num_or_size_splits=2, axis=2)
        top_left, top_right = tf.split(top, num_or_size_splits=2, axis=3)
        bottom_left, bottom_right = tf.split(bottom, num_or_size_splits=2, axis=3)

        top_shift = tf.concat(axis=3, values=[bottom_right, bottom_left])
        bottom_shift = tf.concat(axis=3, values=[top_right, top_left])
        shifted_image = tf.concat(axis=2, values=[top_shift, bottom_shift])


        # Shifting
        top_imag, bottom_imag = tf.split(rec_image_imag, num_or_size_splits=2, axis=2)
        top_left_imag, top_right_imag = tf.split(top_imag, num_or_size_splits=2, axis=3)
        bottom_left_imag, bottom_right_imag = tf.split(bottom_imag, num_or_size_splits=2, axis=3)

        top_shift_imag = tf.concat(axis=3, values=[bottom_right_imag, bottom_left_imag])
        bottom_shift_imag = tf.concat(axis=3, values=[top_right_imag, top_left_imag])
        shifted_image_imag = tf.concat(axis=2, values=[top_shift_imag, bottom_shift_imag])

        shifted_image_two_channels = tf.stack([shifted_image[:,0,:,:], shifted_image_imag[:,0,:,:]], axis=1)
        return shifted_image_two_channels
Ejemplo n.º 4
0
    def step_loss(self, state, action, time):
        # cost:
        x_h = tf.slice(state, [0, self.x_h_field[0]], [-1, 1])
        x_t = tf.slice(state, [0, self.x_t_field[0]], [-1, self.n_t])

        # 0. smooth acceleration policy
        cost_accel = tf.square(action)
        cost_accel_d = tf.mul(tf.pow(self.gamma, time), cost_accel)

        # 1. forcing the host to move forward (until the right point of the roundabout)
        cost_prog = tf.square(self.x_goal - x_h)
        cost_prog_d = tf.mul(tf.pow(self.gamma, time), cost_prog)
        cost_prog_d = tf.squeeze(cost_prog_d, squeeze_dims=[1])

        # 2. keeping distance from vehicles ahead
        # distance to other vehicles
        x_abs_diffs = tf.abs(x_h - x_t)

        # punish only vehicles closer than "require distance"
        cost_acci = tf.nn.relu(self.require_distance - x_abs_diffs)

        # punish only w.r.t vehicles ahead
        cost_acci = tf.mul(cost_acci, tf.to_float(x_h < x_t))

        # sum over all vehicles
        cost_acci = tf.reduce_sum(cost_acci)

        # punish only when host is inside the roundabout (or very close to enter)
        cost_acci = tf.mul(cost_acci, tf.to_float(x_h > -0.5 * self.host_length))

        cost_acci_d = tf.mul(tf.pow(self.gamma, time), cost_acci)
        cost_acci_d = tf.squeeze(cost_acci_d, squeeze_dims=[1])

        return tf.transpose(tf.pack(values=[cost_accel_d, cost_prog_d, cost_acci_d], name='scan_return'))
Ejemplo n.º 5
0
  def construct_embedding(self):
    """Builds an embedding function on top of images.

    Method to be overridden by implementations.

    Returns:
      embeddings: A 2-d float32 `Tensor` of shape [batch_size, embedding_size]
        holding the embedded images.
    """
    with tf.variable_scope('tcn_net', reuse=self._reuse) as vs:
      self._adaptation_scope = vs.name
      net = self._pretrained_output

      # Define some adaptation blocks on top of the pre-trained resnet output.
      adaptation_blocks = []
      adaptation_block_params = [map(
          int, i.split('_')) for i in self._config.adaptation_blocks.split('-')]
      for i, (depth, num_units) in enumerate(adaptation_block_params):
        block = resnet_v2.resnet_v2_block(
            'adaptation_block_%d' % i, base_depth=depth, num_units=num_units,
            stride=1)
        adaptation_blocks.append(block)

      # Stack them on top of the resent output.
      net = resnet_utils.stack_blocks_dense(
          net, adaptation_blocks, output_stride=None)

      # Average pool the output.
      net = tf.reduce_mean(net, [1, 2], name='adaptation_pool', keep_dims=True)

      if self._config.emb_connection == 'fc':
        # Use fully connected layer to project to embedding layer.
        fc_hidden_sizes = self._config.fc_hidden_sizes
        if fc_hidden_sizes == 'None':
          fc_hidden_sizes = []
        else:
          fc_hidden_sizes = map(int, fc_hidden_sizes.split('_'))
        fc_hidden_keep_prob = self._config.dropout.keep_fc
        net = tf.squeeze(net)
        for fc_hidden_size in fc_hidden_sizes:
          net = slim.layers.fully_connected(net, fc_hidden_size)
          if fc_hidden_keep_prob < 1.0:
            net = slim.dropout(net, keep_prob=fc_hidden_keep_prob,
                               is_training=self._is_training)

        # Connect last FC layer to embedding.
        embedding = slim.layers.fully_connected(net, self._embedding_size,
                                                activation_fn=None)
      else:
        # Use 1x1 conv layer to project to embedding layer.
        embedding = slim.conv2d(
            net, self._embedding_size, [1, 1], activation_fn=None,
            normalizer_fn=None, scope='embedding')
        embedding = tf.squeeze(embedding)

      # Optionally L2 normalize the embedding.
      if self._embedding_l2:
        embedding = tf.nn.l2_normalize(embedding, dim=1)

      return embedding
Ejemplo n.º 6
0
    def __init__(self, memory_cells, query, project_query=False):
        """Define Attention.

        Args:
            memory_cells (SequenceBatch): a SequenceBatch containing a Tensor of shape (batch_size, num_cells, cell_dim)
            query (Tensor): a tensor of shape (batch_size, query_dim).
            project_query (bool): defaults to False. If True, the query goes through an extra projection layer to
                coerce it to cell_dim.
        """
        cell_dim = memory_cells.values.get_shape().as_list()[2]
        if project_query:
            # project the query up/down to cell_dim
            self._projection_layer = Dense(cell_dim, activation='linear')
            query = self._projection_layer(query)  # (batch_size, cand_dim)

        memory_values, memory_mask = memory_cells.values, memory_cells.mask

        # batch matrix multiply to compute logit scores for all choices in all batches
        query = tf.expand_dims(query, 2)  # (batch_size, cell_dim, 1)
        logit_values = tf.batch_matmul(memory_values, query)  # (batch_size, num_cells, 1)
        logit_values = tf.squeeze(logit_values, [2])  # (batch_size, num_cells)

        # set all pad logits to negative infinity
        logits = SequenceBatch(logit_values, memory_mask)
        logits = logits.with_pad_value(-float('inf'))

        # normalize to get probs
        probs = tf.nn.softmax(logits.values)  # (batch_size, num_cells)

        retrieved = tf.batch_matmul(tf.expand_dims(probs, 1), memory_values)  # (batch_size, 1, cell_dim)
        retrieved = tf.squeeze(retrieved, [1])  # (batch_size, cell_dim)

        self._logits = logits.values
        self._probs = probs
        self._retrieved = retrieved
Ejemplo n.º 7
0
    def entropy(self, n, p):
        # Note that given n and p where p is a probability vector of
        # length k, the entropy requires a sum over all
        # possible configurations of a k-vector which sums to n. It's
        # expensive.
        # http://stackoverflow.com/questions/36435754/generating-a-numpy-array-with-all-combinations-of-numbers-that-sum-to-less-than
        sess = tf.Session()
        n = sess.run(tf.cast(tf.squeeze(n), dtype=tf.int32))
        sess.close()
        p = tf.cast(tf.squeeze(p), dtype=tf.float32)
        if isinstance(n, np.int32):
            k = get_dims(p)[0]
            max_range = np.zeros(k, dtype=np.int32) + n
            x = np.array([i for i in product(*(range(i+1) for i in max_range))
                                 if sum(i)==n])
            logpmf = self.logpmf(x, n, p)
            return tf.reduce_sum(tf.mul(tf.exp(logpmf), logpmf))
        else:
            out = []
            for j in range(n.shape[0]):
                k = get_dims(p)[0]
                max_range = np.zeros(k, dtype=np.int32) + n[j]
                x = np.array([i for i in product(*(range(i+1) for i in max_range))
                                     if sum(i)==n[j]])
                logpmf = self.logpmf(x, n[j], p[j, :])
                out += [tf.reduce_sum(tf.mul(tf.exp(logpmf), logpmf))]

            return tf.pack(out)
Ejemplo n.º 8
0
  def testExpandAndSqueeze(self):
    with self.cached_session():

      # TODO(aselle): sparse_split, sparse_reduce_sum,
      #  sparse_reduce_sum_sparse, reduce_join
      a = [[1, 2, 3]]
      self.assertAllEqual(tf.expand_dims(tf.squeeze(a, [0]), 0).eval(),
                          a)
      self.assertAllEqual(tf.squeeze(tf.expand_dims(a, 1), [1]).eval(),
                          a)
      self.assertAllEqual(
          tf.expand_dims(
              tf.squeeze(
                  [[1, 2, 3]], squeeze_dims=[0]), dim=0).eval(),
          a)
      self.assertAllEqual(
          tf.squeeze(
              tf.expand_dims(
                  [[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(),
          a)

      self.assertAllEqual(
          tf.squeeze(
              tf.expand_dims(
                  [[1, 2, 3]], dim=1), squeeze_dims=[1]).eval(),
          a)
Ejemplo n.º 9
0
    def _inference(self, x, site, dropout):
        # Get each image from the pair
        print(x.get_shape())
        x_0 = tf.squeeze(x[:, :, :, 0])
        x_1 = tf.squeeze(x[:, :, :, 1])

        # Share weights between the two models of the pair
        with tf.variable_scope("siamese") as scope:
            model0 = self.build_model(x_0)
            scope.reuse_variables()
            model1 = self.build_model(x_1)

        # Dot product layer
        x = self.corr_layer(model0, model1)

        N, M, F = x.get_shape()
        x = tf.reshape(x, [int(N), int(M*F)])

        site = tf.expand_dims(site, axis=1)
        x = tf.concat(1, [x, site])

        for i, M in enumerate(self.M[:-1]):
            with tf.variable_scope('fc{}'.format(i + 1)):
                x = tf.nn.dropout(x, dropout)
                x = self.fc(x, M)

        # Logits linear layer
        with tf.variable_scope('logits'):
            x = tf.nn.dropout(x, dropout)
            x = self.fc(x, self.M[-1], relu=False)

        return tf.squeeze(x)  # tf.sigmoid(x)
Ejemplo n.º 10
0
 def logpmf(self, x, n, p):
     x = tf.cast(tf.squeeze(x), dtype=tf.float32)
     n = tf.cast(tf.squeeze(n), dtype=tf.float32)
     p = tf.cast(tf.squeeze(p), dtype=tf.float32)
     return log_gamma(n + 1.0) - log_gamma(x + 1.0) - \
            log_gamma(n - x + 1.0) + \
            tf.mul(x, tf.log(p)) + tf.mul(n - x, tf.log(1.0-p))
Ejemplo n.º 11
0
  def testSlowVsFast(self):
    model, features = get_model(transformer.transformer_small())

    decode_length = 3

    out_logits, _ = model(features)
    out_logits = tf.squeeze(out_logits, axis=[2, 3])
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=tf.reshape(out_logits, [-1, VOCAB_SIZE]),
        labels=tf.reshape(features["targets"], [-1]))
    loss = tf.reduce_mean(loss)
    apply_grad = tf.train.AdamOptimizer(0.001).minimize(loss)

    with self.test_session():
      tf.global_variables_initializer().run()
      for _ in range(100):
        apply_grad.run()

    model.set_mode(tf.estimator.ModeKeys.PREDICT)

    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
      greedy_result = model._slow_greedy_infer(
          features, decode_length)["outputs"]
      greedy_result = tf.squeeze(greedy_result, axis=[2, 3])

      fast_result = model._greedy_infer(features, decode_length)["outputs"]

    with self.test_session():
      greedy_res = greedy_result.eval()
      fast_res = fast_result.eval()

    self.assertEqual(fast_res.shape, (BATCH_SIZE, INPUT_LENGTH + decode_length))
    self.assertAllClose(greedy_res, fast_res)
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
    correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
    correctly_recognised_indices = tf.squeeze(tf.where(correct_prediction), [1])  # indices of correctly recognised images
    incorrectly_recognised_indices = tf.squeeze(tf.where(tf.logical_not(correct_prediction)), [1]) # indices of incorrectly recognised images
    everything_incorrect_first = tf.concat([incorrectly_recognised_indices, correctly_recognised_indices], 0) # images reordered with indeces of unrecognised images first
    everything_incorrect_first = tf.slice(everything_incorrect_first, [0], [n]) # compute first 100 only - no space to display more anyway
    # compute n=100 digits to display only
    Xs = tf.gather(X, everything_incorrect_first)
    Ys = tf.gather(Y, everything_incorrect_first)
    Ys_ = tf.gather(Y_, everything_incorrect_first)
    correct_prediction_s = tf.gather(correct_prediction, everything_incorrect_first)

    digits_left = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_left())
    correct_tags = tf.gather(digits_left, tf.argmax(Ys_, 1)) # correct digits to be printed on the images
    digits_right = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_right())
    computed_tags = tf.gather(digits_right, tf.argmax(Ys, 1)) # computed digits to be printed on the images
    #superimposed_digits = correct_tags+computed_tags
    superimposed_digits = tf.where(correct_prediction_s, tf.zeros_like(correct_tags),correct_tags+computed_tags) # only pring the correct and computed digits on unrecognised images
    correct_bkg   = tf.reshape(tf.tile([1.3,1.3,1.3], [28*28]), [1, 28,28,3]) # white background
    incorrect_bkg = tf.reshape(tf.tile([1.3,1.0,1.0], [28*28]), [1, 28,28,3]) # red background
    recognised_bkg = tf.gather(tf.concat([incorrect_bkg, correct_bkg], 0), tf.cast(correct_prediction_s, tf.int32)) # pick either the red or the white background depending on recognised status

    I = tf.image.grayscale_to_rgb(Xs)
    I = ((1-(I+superimposed_digits))*recognised_bkg)/1.3 # stencil extra data on top of images and reorder them unrecognised first
    I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
    Islices = [] # 100 images => 10x10 image block
    for imslice in range(lines):
        Islices.append(tf.concat(tf.unstack(tf.slice(I, [imslice*n//lines,0,0,0], [n//lines,28,28,3])), 1))
    I = tf.concat(Islices, 0)
    return I
Ejemplo n.º 13
0
def inference_input():
    """Returns ops that convert raw image data to a 4D tensor representing a single image.

    Taken from:
    https://github.com/tensorflow/serving/blob/master/tensorflow_serving/example/inception_export.py

    The input to the first op can be read using:
    tf.gfile.FastGFile(image_filename, 'r').read()

    """
    # Decode image into float range [0,1]
    jpegs = tf.placeholder(tf.string, shape=(1), name='input')
    image_buffer = tf.squeeze(jpegs, [0])
    image = tf.image.decode_jpeg(image_buffer, channels=3)
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    image = tf.image.central_crop(image, central_fraction=0.875)
    image = tf.expand_dims(image, 0)
    image = tf.image.resize_bilinear(image, [FLAGS.image_size, FLAGS.image_size], align_corners=False)
    image = tf.squeeze(image, [0])

    # Rescale the image to [-1,-1]
    image = tf.sub(image, 0.5)
    image = tf.mul(image, 2.0)
    images = tf.expand_dims(image, 0)

    return images, jpegs
Ejemplo n.º 14
0
 def _log_unnormalized_prob(self, x):
   mean = tf.squeeze(tf.gather(x, [0], axis=-1), axis=-1)
   precision = self._maybe_assert_valid_sample(
       tf.squeeze(tf.gather(x, [1], axis=-1), axis=-1))
   return (tf.math.xlogy(self.concentration - 0.5, precision)
           - self.rate * precision
           - 0.5 * self._lambda * precision * tf.square(mean - self.loc))
Ejemplo n.º 15
0
def preprocess_for_test(image, gt_boxes, gt_masks):


    ih, iw = tf.shape(image)[0], tf.shape(image)[1]

    ## min size resizing
    new_ih, new_iw = preprocess_utils._smallest_size_at_least(ih, iw, cfg.FLAGS.image_min_size)
    image = tf.expand_dims(image, 0)
    image = tf.image.resize_bilinear(image, [new_ih, new_iw], align_corners=False)
    image = tf.squeeze(image, axis=[0])

    gt_masks = tf.expand_dims(gt_masks, -1)
    gt_masks = tf.cast(gt_masks, tf.float32)
    gt_masks = tf.image.resize_nearest_neighbor(gt_masks, [new_ih, new_iw], align_corners=False)
    gt_masks = tf.cast(gt_masks, tf.int32)
    gt_masks = tf.squeeze(gt_masks, axis=[-1])

    scale_ratio = tf.to_float(new_ih) / tf.to_float(ih)
    gt_boxes = preprocess_utils.resize_gt_boxes(gt_boxes, scale_ratio)
    
    ## zero mean image
    image = tf.cast(image, tf.float32)
    image = image / 256.0
    image = (image - 0.5) * 2.0
    image = tf.expand_dims(image, axis=0)

    ## rgb to bgr
    image = tf.reverse(image, axis=[-1])

    return image, gt_boxes, gt_masks 
    def _forward(self, obs_prob_list):
        
        with tf.name_scope('init_scaling_factor'):
            self.scale = tf.Variable(tf.zeros([self.N], tf.float64)) #scale factors
        
        with tf.name_scope('forward_first_step'):
            # initialize with state starting priors
            init_prob = tf.mul(self.T0, tf.squeeze(obs_prob_list[0]))

            # scaling factor at t=0
            self.scale = tf.scatter_update(self.scale, 0, 1.0 / tf.reduce_sum(init_prob))

            # scaled belief at t=0
            self.forward = tf.scatter_update(self.forward, 0, self.scale[0] * init_prob)

        # propagate belief
        for step, obs_prob in enumerate(obs_prob_list[1:]):
            with tf.name_scope('time_step-%s' %step):
                # previous state probability
                prev_prob = tf.expand_dims(self.forward[step, :], 0)
                # transition prior
                prior_prob = tf.matmul(prev_prob, self.T)
                # forward belief propagation
                forward_score = tf.mul(prior_prob, tf.squeeze(obs_prob))

                forward_prob = tf.squeeze(forward_score)
                # scaling factor
                self.scale = tf.scatter_update(self.scale, step+1, 1.0 / tf.reduce_sum(forward_prob))
                # Update forward matrix
                self.forward = tf.scatter_update(self.forward, step+1, self.scale[step+1] * forward_prob)
Ejemplo n.º 17
0
 def convolution(self, inputs, num_units):
     x = tf.expand_dims(inputs, 3)
     chan_in = 1
     
     #Bigram
     w_bigram = tf.get_variable("w_bigram", shape= [2,50,chan_in,num_units],
                              initializer= tf.contrib.layers.xavier_initializer_conv2d())
     b_bigram = tf.get_variable("b_bigram", shape= [num_units])
     y_bigram = self.nonlin(tf.nn.conv2d(x, w_bigram, strides= [1,1,1,1], padding='VALID') + b_bigram)
     h_bigram = tf.reduce_max(tf.squeeze(y_bigram) , 1)
     
     #Trigram
     w_trigram = tf.get_variable("w_trigram", shape= [3,50,chan_in,num_units],
                              initializer= tf.contrib.layers.xavier_initializer_conv2d())
     b_trigram = tf.get_variable("b_trigram", shape= [num_units])
     y_trigram = self.nonlin(tf.nn.conv2d(x, w_trigram, strides= [1,1,1,1], padding='VALID') + b_trigram)
     h_trigram = tf.reduce_max(tf.squeeze(y_trigram) , 1)
     
     #Quin-gram
     w_quingram = tf.get_variable("w_quingram", shape= [3,50,chan_in,num_units],
                              initializer= tf.contrib.layers.xavier_initializer_conv2d())
     b_quingram = tf.get_variable("b_quingram", shape= [num_units])
     y_quingram = self.nonlin(tf.nn.conv2d(x, w_trigram, strides= [1,1,1,1], padding='VALID') + b_trigram)
     h_quingram = tf.reduce_max(tf.squeeze(y_quingram) , 1)
     
     if self.hyperparams['conv_type'] == 'bigram':
         h = h_bigram
     elif self.hyperparams['conv_type'] == 'trigram':
         h = h_trigram
     elif self.hyperparams['conv_type'] == 'quingram':
         h = h_quingram            
     elif self.hyperparams['conv_type'] == 'inception':
         h = tf.concat(1, [h_bigram, h_trigram, h_quingram])
         
     return h
Ejemplo n.º 18
0
def filename2image(image_size=299, central_fraction=0.875):
    """Returns ops that convert a filename to a 4D tensor representing a single image.

    Usage:
        image_op, buffer_op = decode_filename()
        imbuffer = tf.gfile.FastGFile('path/to/file.jpg', 'r').read()
        with tf.Session() as sess:
            image = sess.run([image_op], feed_dict={buffer_op : [imbuffer]})
        plt.imshow(image)


    Taken from:
    https://github.com/tensorflow/serving/blob/master/tensorflow_serving/example/inception_export.py

    Args:
        image_size (int): image is resized to this (default 299 for Inception v3)
        central_fraction (float): the central fraction crop to take

    Returns:
        Two ops, one for the image buffer input, the other the tensorflow-ready image.

    """
    image_buffer = tf.placeholder(tf.string, shape=(1))
    image_buffer = tf.squeeze(image_buffer, [0])
    image = tf.image.decode_jpeg(image_buffer, channels=3)
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    image = tf.image.central_crop(image, central_fraction)
    image = tf.expand_dims(image, 0)
    image = tf.image.resize_bilinear(image, [image_size, image_size], align_corners=False)
    image = tf.squeeze(image, [0])
    image = tf.sub(image, 0.5)
    image = tf.mul(image, 2.0)
    image = tf.expand_dims(image, 0)

    return image, image_buffer
Ejemplo n.º 19
0
    def attention(self,ref, query, with_softmax, scope="attention"):
        """

        :param ref: encoder的输出
        :param query: decoder的输出
        :param with_softmax:
        :param scope:
        :return:
        """
        with tf.variable_scope(scope):
            W_1 = tf.get_variable("W_e", [self.hidden_dim, self.attention_dim], initializer=self.initializer)  # L x A
            W_2 = tf.get_variable("W_d", [self.hidden_dim, self.attention_dim], initializer=self.initializer) # L * A

            dec_portion = tf.matmul(query, W_2)

            scores = [] # S * B
            v_blend = tf.get_variable("v_blend", [self.attention_dim, 1], initializer=self.initializer)  # A x 1
            bais_blend = tf.get_variable("bais_v_blend", [1], initializer=self.initializer)  # 1 x 1
            for i in range(self.max_enc_length + 1):
                refi = tf.matmul(tf.squeeze(ref[:,i,:]),W_1)
                ui = tf.add(tf.matmul(tf.nn.tanh(dec_portion+refi),v_blend),bais_blend) # B * 1
                scores.append(tf.squeeze(ui))
            scores = tf.transpose(scores,[1,0]) # B * S
            if with_softmax:
                return tf.nn.softmax(scores,dim=1)
            else:
                return scores
Ejemplo n.º 20
0
  def compute_accuracy(x, l, mask):
    """Compute model accuracy."""
    preds = ch_model.get_probs(x)
    preds = tf.squeeze(preds)
    preds = tf.argmax(preds, -1, output_type=l.dtype)

    _, acc_update_op = tf.metrics.accuracy(l, preds, weights=mask)

    if FLAGS.surrogate_attack:
      preds = sur_ch_model.get_probs(x)
      preds = tf.squeeze(preds)
      preds = tf.argmax(preds, -1, output_type=l.dtype)
      acc_update_op = tf.tuple((acc_update_op,
                                tf.metrics.accuracy(l, preds, weights=mask)[1]))

    sess.run(tf.initialize_local_variables())
    for i in range(FLAGS.eval_steps):
      tf.logging.info(
          "\tEvaluating batch [%d / %d]" % (i + 1, FLAGS.eval_steps))
      acc = sess.run(acc_update_op)
    if FLAGS.surrogate_attack:
      tf.logging.info("\tFinal acc: (%.4f, %.4f)" % (acc[0], acc[1]))
    else:
      tf.logging.info("\tFinal acc: %.4f" % acc)
    return acc
def hnet_transformation(gt_pts, transformation_coeffcient, name):
    """

    :param gt_pts:
    :param transformation_coeffcient:
    :param name:
    :return:
    """
    with tf.variable_scope(name):
        # 首先映射原始标签点对
        transformation_coeffcient = tf.concat([transformation_coeffcient, [1.0]], axis=-1)
        H_indices = tf.constant([[0], [1], [2], [4], [5], [7], [8]])
        H_shape = tf.constant([9])
        H = tf.scatter_nd(H_indices, transformation_coeffcient, H_shape)
        H = tf.reshape(H, shape=[3, 3])

        gt_pts = tf.transpose(gt_pts)
        pts_projects = tf.matmul(H, gt_pts)

        # 求解最小二乘二阶多项式拟合参数矩阵
        Y = tf.transpose(pts_projects[1, :])
        X = tf.transpose(pts_projects[0, :])
        Y_One = tf.add(tf.subtract(Y, Y), tf.constant(1.0, tf.float32))
        Y_stack = tf.stack([tf.pow(Y, 3), tf.pow(Y, 2), Y, Y_One], axis=1)
        w = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(Y_stack), Y_stack)),
                                tf.transpose(Y_stack)),
                      tf.expand_dims(X, -1))

        # 利用二阶多项式参数求解拟合位置
        x_preds = tf.matmul(Y_stack, w)
        preds = tf.transpose(tf.stack([tf.squeeze(x_preds, -1), Y, Y_One], axis=1))
        preds_fit = tf.stack([tf.squeeze(x_preds, -1), Y], axis=1)
        x_transformation_back = tf.matmul(tf.matrix_inverse(H), preds)

    return x_transformation_back
    def unpool_layer2x2(self, x, raveled_argmax, out_shape):
        argmax = self.unravel_argmax(raveled_argmax, tf.to_int64(out_shape))
        output = tf.zeros([out_shape[1], out_shape[2], out_shape[3]])

        height = tf.shape(output)[0]
        width = tf.shape(output)[1]
        channels = tf.shape(output)[2]

        t1 = tf.to_int64(tf.range(channels))
        t1 = tf.tile(t1, [((width + 1) // 2) * ((height + 1) // 2)])
        t1 = tf.reshape(t1, [-1, channels])
        t1 = tf.transpose(t1, perm=[1, 0])
        t1 = tf.reshape(t1, [channels, (height + 1) // 2, (width + 1) // 2, 1])

        t2 = tf.squeeze(argmax)
        t2 = tf.pack((t2[0], t2[1]), axis=0)
        t2 = tf.transpose(t2, perm=[3, 1, 2, 0])

        t = tf.concat(3, [t2, t1])
        indices = tf.reshape(t, [((height + 1) // 2) * ((width + 1) // 2) * channels, 3])

        x1 = tf.squeeze(x)
        x1 = tf.reshape(x1, [-1, channels])
        x1 = tf.transpose(x1, perm=[1, 0])
        values = tf.reshape(x1, [-1])

        delta = tf.SparseTensor(indices, values, tf.to_int64(tf.shape(output)))
        return tf.expand_dims(tf.sparse_tensor_to_dense(tf.sparse_reorder(delta)), 0)
Ejemplo n.º 23
0
def randomly_scale_image_and_label(image, label=None, scale=1.0):
  """Randomly scales image and label.

  Args:
    image: Image with shape [height, width, 3].
    label: Label with shape [height, width, 1].
    scale: The value to scale image and label.

  Returns:
    Scaled image and label.
  """
  # No random scaling if scale == 1.
  if scale == 1.0:
    return image, label
  image_shape = tf.shape(image)
  new_dim = tf.cast(
      tf.cast([image_shape[0], image_shape[1]], tf.float32) * scale,
      tf.int32)

  # Need squeeze and expand_dims because image interpolation takes
  # 4D tensors as input.
  image = tf.squeeze(tf.image.resize_bilinear(
      tf.expand_dims(image, 0),
      new_dim,
      align_corners=True), [0])
  if label is not None:
    label = tf.squeeze(tf.image.resize_nearest_neighbor(
        tf.expand_dims(label, 0),
        new_dim,
        align_corners=True), [0])

  return image, label
Ejemplo n.º 24
0
    def build_detector(self):
        img_size = self.config['image_size']
        self.image_ph = tf.placeholder(shape=[None, None, 3],
                                       dtype=tf.float32, name='img_ph')
        self.seg_ph = tf.placeholder(shape=[None, None], dtype=tf.int32, name='seg_ph')

        img = tf.image.resize_bilinear(tf.expand_dims(self.image_ph, 0),
                                       (img_size, img_size))
        self.net.create_trunk(img)

        if args.detect:
            self.net.create_multibox_head(self.loader.num_classes)
            confidence = tf.nn.softmax(tf.squeeze(self.net.outputs['confidence']))
            location = tf.squeeze(self.net.outputs['location'])
            self.nms(location, confidence, self.bboxer.tiling)

        if args.segment:
            self.net.create_segmentation_head(self.loader.num_classes)
            self.segmentation = self.net.outputs['segmentation']
            seg_shape = tf.shape(self.image_ph)[:2]
            self.segmentation = tf.image.resize_bilinear(self.segmentation, seg_shape)

            self.segmentation = tf.cast(tf.argmax(tf.squeeze(self.segmentation), axis=-1), tf.int32)
            self.segmentation = tf.reshape(self.segmentation, seg_shape)
            self.segmentation.set_shape([None, None])

            if not self.no_gt:
                easy_mask = self.seg_ph <= self.loader.num_classes
                predictions = tf.boolean_mask(self.segmentation, easy_mask)
                labels = tf.boolean_mask(self.seg_ph, easy_mask)
                self.mean_iou, self.iou_update = mean_iou(predictions, labels, self.loader.num_classes)
            else:
                self.mean_iou = tf.constant(0)
                self.iou_update = tf.constant(0)
Ejemplo n.º 25
0
	def buildConvolution(self):
		q_embedding = self.tensors['q_embedding']
		a_embedding = self.tensors['a_embedding']
		with tf.name_scope('convolution'):
			filter_shape = (self.params['filters'][0], self.wdim, 1, self.params['nb_filter'])
			W = glorot_normal(filter_shape, name="W")
			b = tf.Variable(tf.constant(0.0, shape=(self.params['nb_filter'],)), name="b")
			q_conv = tf.nn.conv2d(
				tf.expand_dims(q_embedding, -1),
				W,
				strides=[1,1,1,1],
				padding="VALID",
				name="q_conv"
			)
			a_conv = tf.nn.conv2d(
				tf.expand_dims(a_embedding, -1),
				W,
				strides=[1,1,1,1],
				padding="VALID",
				name = "a_conv"
			)
			q_conv = tf.squeeze(q_conv, [2])
			a_conv = tf.squeeze(a_conv, [2])
			# shape = (batch, q_length, NUM_FILTERS)
			q_relu = tf.nn.relu(tf.nn.bias_add(q_conv, b), name="q_relu")
			# shape = (batch, a_length, NUM_FILTERS)
			a_relu = tf.nn.relu(tf.nn.bias_add(a_conv, b), name="q_relu")
		self.tensors['q_conv'] = q_conv
		self.tensors['a_conv'] = a_conv
		self.tensors['q_relu'] = q_relu
		self.tensors['a_relu'] = a_relu
		self.tensors.setdefault('weights', []).append(b)
		self.tensors.setdefault('summary', []).append(tf.nn.zero_fraction(a_relu))
Ejemplo n.º 26
0
  def _match(self, similarity_matrix, valid_rows):
    """Bipartite matches a collection rows and columns. A greedy bi-partite.

    TODO(rathodv): Add num_valid_columns options to match only that many columns
    with all the rows.

    Args:
      similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
        where higher values mean more similar.
      valid_rows: A boolean tensor of shape [N] indicating the rows that are
        valid.

    Returns:
      match_results: int32 tensor of shape [M] with match_results[i]=-1
        meaning that column i is not matched and otherwise that it is matched to
        row match_results[i].
    """
    valid_row_sim_matrix = tf.gather(similarity_matrix,
                                     tf.squeeze(tf.where(valid_rows), axis=-1))
    invalid_row_sim_matrix = tf.gather(
        similarity_matrix,
        tf.squeeze(tf.where(tf.logical_not(valid_rows)), axis=-1))
    similarity_matrix = tf.concat(
        [valid_row_sim_matrix, invalid_row_sim_matrix], axis=0)
    # Convert similarity matrix to distance matrix as tf.image.bipartite tries
    # to find minimum distance matches.
    distance_matrix = -1 * similarity_matrix
    num_valid_rows = tf.reduce_sum(tf.to_float(valid_rows))
    _, match_results = image_ops.bipartite_match(
        distance_matrix, num_valid_rows=num_valid_rows)
    match_results = tf.reshape(match_results, [-1])
    match_results = tf.cast(match_results, tf.int32)
    return match_results
Ejemplo n.º 27
0
  def _add_box_predictions_to_feature_maps(self, feature_maps):
    """Adds box predictors to each feature map and returns concatenated results.

    Args:
      feature_maps: a list of tensors where the ith tensor has shape
        [batch, height_i, width_i, depth_i]

    Returns:
      box_encodings: 4-D float tensor of shape [batch_size, num_anchors,
          box_code_dimension] containing predicted boxes.
      class_predictions_with_background: 2-D float tensor of shape
          [batch_size, num_anchors, num_classes+1] containing class predictions
          (logits) for each of the anchors.  Note that this tensor *includes*
          background class predictions (at class index 0).

    Raises:
      RuntimeError: if the number of feature maps extracted via the
        extract_features method does not match the length of the
        num_anchors_per_locations list that was passed to the constructor.
      RuntimeError: if box_encodings from the box_predictor does not have
        shape of the form  [batch_size, num_anchors, 1, code_size].
    """
    num_anchors_per_location_list = 1
    if len(feature_maps) != len(num_anchors_per_location_list):
      raise RuntimeError('the number of feature maps must match the '
                         'length of self.anchors.NumAnchorsPerLocation().')
    box_encodings_list = []
    mask_encodings_list = []
    for idx, (feature_map, num_anchors_per_location
             ) in enumerate(zip(feature_maps, num_anchors_per_location_list)):
      box_predictor_scope = 'BoxPredictor_{}'.format(idx)
      box_predictions = self._box_predictor.predict(feature_map,
                                                    num_anchors_per_location,
                                                    box_predictor_scope)
      box_encodings = box_predictions[bpredictor.BOX_ENCODINGS]
      mask_encodings = box_predictions[bpredictor.MASK_PREDICTIONS]

      box_encodings_shape = box_encodings.get_shape().as_list()
      if len(box_encodings_shape) != 5 or box_encodings_shape[2] != 1:
        raise RuntimeError('box_encodings from the box_predictor must be of '
                           'shape `[batch_size, num_anchors, 1, code_size]`; '
                           'actual shape', box_encodings_shape)
      box_encodings = tf.squeeze(box_encodings, axis=2)
      mask_encodings = tf.squeeze(mask_encodings, axis=2)
      box_encodings_list.append(box_encodings)
      mask_encodings_list.append(mask_encodings)
    
    """ 
    num_predictions = sum(
        [tf.shape(box_encodings)[1] for box_encodings in box_encodings_list])
    num_anchors = self.anchors.num_boxes()
    anchors_assert = tf.assert_equal(num_anchors, num_predictions, [
        'Mismatch: number of anchors vs number of predictions', num_anchors,
        num_predictions
    ])
    """
    
    box_encodings = tf.concat(box_encodings_list, 1)
    mask_encodings = tf.concat(mask_encodings_list, 1)
    return box_encodings, mask_encodings
Ejemplo n.º 28
0
    def build_network(self):
        net_tensors = self.net_tensors
        with self.net_graph.as_default(), tf.device(self.net_device):
            logits = tf.placeholder(dtype=tf.float32, shape=(self.batch_size, self.image_classes))
            labels = tf.placeholder(dtype=tf.int32, shape=(self.batch_size,))
            lambs = tf.placeholder(dtype=tf.float32, shape=(self.image_classes,))
            # put a sigfunction on logits and then transpose
            logits = tf.transpose(framwork.sig_func(logits))
            # according to the labels, erase rows which is not in labels

            labels_unique = tf.constant(range(self.image_classes), dtype=tf.int32)
            labels_num = self.image_classes
            logits = tf.gather(logits, indices=labels_unique)
            lambs = tf.gather(lambs, indices=labels_unique)
            # set the value of each row to True when it occurs in labels
            templete = tf.tile(tf.expand_dims(labels_unique, dim=1), [1, self.batch_size])
            labels_expand = tf.tile(tf.expand_dims(labels, dim=0), [labels_num, 1])
            indict_logic = tf.equal(labels_expand, templete)
            # split the tensor along rows
            logit_list = tf.split(0, labels_num, logits)
            indict_logic_list = tf.split(0, labels_num, indict_logic)
            lamb_list = tf.split(0, self.image_classes, lambs)
            logit_list = [tf.squeeze(item) for item in logit_list]
            indict_logic_list = [tf.squeeze(item) for item in indict_logic_list]
            left_right_tuples = list()
            for i in range(self.image_classes):
                left_right_tuples.append(framwork.lamb_func(logit_list[i], indict_logic_list[i], lamb=lamb_list[i]))
            # func = framwork.lamb_func()
            # left_right_tuples = map(func, logit_list, indict_logic_list, lamb_list)
            net_tensors.update({'left_right_tuples': left_right_tuples, 'logits': logits, 'labels': labels,
                                'lambs': lambs})
Ejemplo n.º 29
0
def conpute_loss(scores, target):
    """ Compute the perplexity of the batch 
    
    Args:
        scores: 3D tensor, shape=(BATCH_SIZE, 1, S_FRENCH, T_FRENCH)
        target: 4D tensor, shape=(BATCH_SIZE, 1, S_FRENCH, T_FRENCH)
        
    Returns:
        tf.float32 tensor
    """
    
    with tf.name_scope('Loss_computation'):
        sortie_loss = tf.squeeze(target)    
        scores = tf.squeeze(scores) 
        
        loss = tf.reduce_sum(tf.mul(scores, sortie_loss), reduction_indices=2) # Get the activation of the target token
        #loss = tf.reduce_sum(loss,reduction_indices=2)
        loss = tf.clip_by_value(loss, clip_value_min=1e-10, clip_value_max=1.0)
        #loss = 
        loss = tf.reduce_sum(tf.log(loss), reduction_indices=1)
        loss = -tf.reduce_mean(loss)
        
        l2_weights = 0.00
        with tf.variable_scope('Embeddings', reuse=True):
            w = tf.get_variable('weights')
            b = tf.get_variable('biases')
            loss = loss + l2_weights*tf.nn.l2_loss(w) + l2_weights*tf.nn.l2_loss(b)
            
        with tf.variable_scope('Decoding', reuse=True):
            w = tf.get_variable('weights')
            b = tf.get_variable('biases')
            loss = loss + l2_weights*tf.nn.l2_loss(w) + l2_weights*tf.nn.l2_loss(b)
        
        return loss
            def get_content_based_attention_vectors(query_vectors):
                '''
                    function that returns the alpha_content vector using the yt-1 (query vectors)
                '''
                # use the W_f and b_f to transform the query_vectors to the shape of f_values
                f_trans_query_vectors = tf.matmul(W_f, tf.transpose(query_vectors)) + b_f
                # use the W_c and b_c to transform the query_vectors to the shape of h_values
                h_trans_query_vectors = tf.matmul(W_c, tf.transpose(query_vectors)) + b_c

                # transpose and expand the dims of the f_trans_query_vectors
                f_trans_query_matrices = tf.expand_dims(tf.transpose(f_trans_query_vectors), axis=-1)
                # obtain the field attention_values by using the matmul operation
                field_attention_values = tf.matmul(tf_field_embedded, f_trans_query_matrices)

                # perform the same process for the h_trans_query_vectors
                h_trans_query_matrices = tf.expand_dims(tf.transpose(h_trans_query_vectors), axis=-1)
                hidden_attention_values = tf.matmul(encoded_input, h_trans_query_matrices)

                # drop the last dimension (1 sized)
                field_attention_values = tf.squeeze(field_attention_values, axis=[-1])
                hidden_attention_values = tf.squeeze(hidden_attention_values, axis=[-1])


                # free up non_required resources:
                ret_value = tf.nn.softmax(field_attention_values * hidden_attention_values, name="softmax")

                # return the element wise multiplied values followed by softmax
                return ret_value
Ejemplo n.º 31
0
def transform_input_data(tensor_dict,
                         model_preprocess_fn,
                         image_resizer_fn,
                         num_classes,
                         data_augmentation_fn=None,
                         merge_multiple_boxes=False,
                         retain_original_image=False,
                         use_multiclass_scores=False,
                         use_bfloat16=False):
  """A single function that is responsible for all input data transformations.

  Data transformation functions are applied in the following order.
  1. If key fields.InputDataFields.image_additional_channels is present in
     tensor_dict, the additional channels will be merged into
     fields.InputDataFields.image.
  2. data_augmentation_fn (optional): applied on tensor_dict.
  3. model_preprocess_fn: applied only on image tensor in tensor_dict.
  4. image_resizer_fn: applied on original image and instance mask tensor in
     tensor_dict.
  5. one_hot_encoding: applied to classes tensor in tensor_dict.
  6. merge_multiple_boxes (optional): when groundtruth boxes are exactly the
     same they can be merged into a single box with an associated k-hot class
     label.

  Args:
    tensor_dict: dictionary containing input tensors keyed by
      fields.InputDataFields.
    model_preprocess_fn: model's preprocess function to apply on image tensor.
      This function must take in a 4-D float tensor and return a 4-D preprocess
      float tensor and a tensor containing the true image shape.
    image_resizer_fn: image resizer function to apply on groundtruth instance
      `masks. This function must take a 3-D float tensor of an image and a 3-D
      tensor of instance masks and return a resized version of these along with
      the true shapes.
    num_classes: number of max classes to one-hot (or k-hot) encode the class
      labels.
    data_augmentation_fn: (optional) data augmentation function to apply on
      input `tensor_dict`.
    merge_multiple_boxes: (optional) whether to merge multiple groundtruth boxes
      and classes for a given image if the boxes are exactly the same.
    retain_original_image: (optional) whether to retain original image in the
      output dictionary.
    use_multiclass_scores: whether to use multiclass scores as
      class targets instead of one-hot encoding of `groundtruth_classes`.
    use_bfloat16: (optional) a bool, whether to use bfloat16 in training.

  Returns:
    A dictionary keyed by fields.InputDataFields containing the tensors obtained
    after applying all the transformations.
  """
  # Reshape flattened multiclass scores tensor into a 2D tensor of shape
  # [num_boxes, num_classes].
  if fields.InputDataFields.multiclass_scores in tensor_dict:
    tensor_dict[fields.InputDataFields.multiclass_scores] = tf.reshape(
        tensor_dict[fields.InputDataFields.multiclass_scores], [
            tf.shape(tensor_dict[fields.InputDataFields.groundtruth_boxes])[0],
            num_classes
        ])
  if fields.InputDataFields.groundtruth_boxes in tensor_dict:
    tensor_dict = util_ops.filter_groundtruth_with_nan_box_coordinates(
        tensor_dict)
    tensor_dict = util_ops.filter_unrecognized_classes(tensor_dict)

  if retain_original_image:
    tensor_dict[fields.InputDataFields.original_image] = tf.cast(
        image_resizer_fn(tensor_dict[fields.InputDataFields.image], None)[0],
        tf.uint8)

  if fields.InputDataFields.image_additional_channels in tensor_dict:
    channels = tensor_dict[fields.InputDataFields.image_additional_channels]
    tensor_dict[fields.InputDataFields.image] = tf.concat(
        [tensor_dict[fields.InputDataFields.image], channels], axis=2)

  # Apply data augmentation ops.
  if data_augmentation_fn is not None:
    tensor_dict = data_augmentation_fn(tensor_dict)

  # Apply model preprocessing ops and resize instance masks.
  image = tensor_dict[fields.InputDataFields.image]
  preprocessed_resized_image, true_image_shape = model_preprocess_fn(
      tf.expand_dims(tf.to_float(image), axis=0))
  if use_bfloat16:
    preprocessed_resized_image = tf.cast(
        preprocessed_resized_image, tf.bfloat16)
  tensor_dict[fields.InputDataFields.image] = tf.squeeze(
      preprocessed_resized_image, axis=0)
  tensor_dict[fields.InputDataFields.true_image_shape] = tf.squeeze(
      true_image_shape, axis=0)
  if fields.InputDataFields.groundtruth_instance_masks in tensor_dict:
    masks = tensor_dict[fields.InputDataFields.groundtruth_instance_masks]
    _, resized_masks, _ = image_resizer_fn(image, masks)
    if use_bfloat16:
      resized_masks = tf.cast(resized_masks, tf.bfloat16)
    tensor_dict[fields.InputDataFields.
                groundtruth_instance_masks] = resized_masks

  # Transform groundtruth classes to one hot encodings.
  label_offset = 1
  zero_indexed_groundtruth_classes = tensor_dict[
      fields.InputDataFields.groundtruth_classes] - label_offset
  tensor_dict[fields.InputDataFields.groundtruth_classes] = tf.one_hot(
      zero_indexed_groundtruth_classes, num_classes)

  if use_multiclass_scores:
    tensor_dict[fields.InputDataFields.groundtruth_classes] = tensor_dict[
        fields.InputDataFields.multiclass_scores]
    tensor_dict.pop(fields.InputDataFields.multiclass_scores, None)

  if fields.InputDataFields.groundtruth_confidences in tensor_dict:
    groundtruth_confidences = tensor_dict[
        fields.InputDataFields.groundtruth_confidences]
    # Map the confidences to the one-hot encoding of classes
    tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
        tf.reshape(groundtruth_confidences, [-1, 1]) *
        tensor_dict[fields.InputDataFields.groundtruth_classes])
  else:
    groundtruth_confidences = tf.ones_like(
        zero_indexed_groundtruth_classes, dtype=tf.float32)
    tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
        tensor_dict[fields.InputDataFields.groundtruth_classes])

  if merge_multiple_boxes:
    merged_boxes, merged_classes, merged_confidences, _ = (
        util_ops.merge_boxes_with_multiple_labels(
            tensor_dict[fields.InputDataFields.groundtruth_boxes],
            zero_indexed_groundtruth_classes,
            groundtruth_confidences,
            num_classes))
    merged_classes = tf.cast(merged_classes, tf.float32)
    tensor_dict[fields.InputDataFields.groundtruth_boxes] = merged_boxes
    tensor_dict[fields.InputDataFields.groundtruth_classes] = merged_classes
    tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
        merged_confidences)
  if fields.InputDataFields.groundtruth_boxes in tensor_dict:
    tensor_dict[fields.InputDataFields.num_groundtruth_boxes] = tf.shape(
        tensor_dict[fields.InputDataFields.groundtruth_boxes])[0]

  return tensor_dict
Ejemplo n.º 32
0
    def __init__(self, args, infer=False):
        self.args = args
        if infer:
            args.batch_size = 1
            args.seq_length = 1

        if args.model == 'rnn':
            cell_fn = rnn.BasicRNNCell
        elif args.model == 'gru':
            cell_fn = rnn.GRUCell
        elif args.model == 'lstm':
            cell_fn = rnn.BasicLSTMCell
        else:
            raise Exception("model type not supported: {}".format(args.model))

        cells = []
        for _ in range(args.num_layers):
            cell = cell_fn(args.rnn_size)
            cells.append(cell)

        self.cell = cell = rnn.MultiRNNCell(cells)

        self.input_data = tf.placeholder(tf.int32,
                                         [args.batch_size, args.seq_length])
        self.targets = tf.placeholder(tf.int32,
                                      [args.batch_size, args.seq_length])
        self.initial_state = cell.zero_state(args.batch_size, tf.float32)
        self.batch_pointer = tf.Variable(0,
                                         name="batch_pointer",
                                         trainable=False,
                                         dtype=tf.int32)
        self.inc_batch_pointer_op = tf.assign(self.batch_pointer,
                                              self.batch_pointer + 1)
        self.epoch_pointer = tf.Variable(0,
                                         name="epoch_pointer",
                                         trainable=False)
        self.batch_time = tf.Variable(0.0, name="batch_time", trainable=False)
        tf.summary.scalar("time_batch", self.batch_time)

        def variable_summaries(var):
            """Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
            with tf.name_scope('summaries'):
                mean = tf.reduce_mean(var)
                tf.summary.scalar('mean', mean)
                #with tf.name_scope('stddev'):
                #   stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
                #tf.summary.scalar('stddev', stddev)
                tf.summary.scalar('max', tf.reduce_max(var))
                tf.summary.scalar('min', tf.reduce_min(var))
                #tf.summary.histogram('histogram', var)

        with tf.variable_scope('rnnlm'):
            softmax_w = tf.get_variable("softmax_w",
                                        [args.rnn_size, args.vocab_size])
            variable_summaries(softmax_w)
            softmax_b = tf.get_variable("softmax_b", [args.vocab_size])
            variable_summaries(softmax_b)
            with tf.device("/cpu:0"):
                embedding = tf.get_variable("embedding",
                                            [args.vocab_size, args.rnn_size])
                inputs = tf.split(
                    tf.nn.embedding_lookup(embedding, self.input_data),
                    args.seq_length, 1)
                inputs = [tf.squeeze(input_, [1]) for input_ in inputs]

        def loop(prev, _):
            prev = tf.matmul(prev, softmax_w) + softmax_b
            prev_symbol = tf.stop_gradient(tf.argmax(prev, 1))
            return tf.nn.embedding_lookup(embedding, prev_symbol)

        outputs, last_state = legacy_seq2seq.rnn_decoder(
            inputs,
            self.initial_state,
            cell,
            loop_function=loop if infer else None,
            scope='rnnlm')
        output = tf.reshape(tf.concat(outputs, 1), [-1, args.rnn_size])
        self.logits = tf.matmul(output, softmax_w) + softmax_b
        self.probs = tf.nn.softmax(self.logits)
        loss = legacy_seq2seq.sequence_loss_by_example(
            [self.logits], [tf.reshape(self.targets, [-1])],
            [tf.ones([args.batch_size * args.seq_length])], args.vocab_size)
        self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length
        tf.summary.scalar("cost", self.cost)
        self.final_state = last_state
        self.lr = tf.Variable(0.0, trainable=False)
        tvars = tf.trainable_variables()
        grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars),
                                          args.grad_clip)
        optimizer = tf.train.AdamOptimizer(self.lr)
        self.train_op = optimizer.apply_gradients(zip(grads, tvars))
def test_classifier(classifier_model_dir: pathlib.Path,
                    fwd_model_dir: List[pathlib.Path],
                    n_actions: int,
                    saved_state: Optional[pathlib.Path],
                    generate_actions: Callable):
    fwd_model, _ = dynamics_utils.load_generic_model([pathlib.Path(p) for p in fwd_model_dir])
    classifier: NNClassifierWrapper = classifier_utils.load_generic_model([classifier_model_dir])

    service_provider = GazeboServices()
    service_provider.setup_env(verbose=0,
                               real_time_rate=0,
                               max_step_size=fwd_model.data_collection_params['max_step_size'],
                               play=False)
    if saved_state:
        service_provider.restore_from_bag(saved_state)

    scenario = fwd_model.scenario
    scenario.on_before_get_state_or_execute_action()

    # NOTE: perhaps it would make sense to have a "fwd_model" have API for get_env, get_state, sample_action, etc
    #  because the fwd_model knows it's scenario, and importantly it also knows it's data_collection_params
    #  which is what we're using here to pass to the scenario methods
    params = fwd_model.data_collection_params
    environment = numpify(scenario.get_environment(params))
    start_state = numpify(scenario.get_state())

    start_state_tiled = repeat(start_state, n_actions, axis=0, new_axis=True)
    start_states_tiled = add_time_dim(start_state_tiled)

    actions = generate_actions(environment, start_state_tiled, scenario, params, n_actions)

    environment_tiled = repeat(environment, n_actions, axis=0, new_axis=True)
    actions_dict = sequence_of_dicts_to_dict_of_tensors(actions)
    actions_dict = add_time_dim(actions_dict)
    predictions, _ = fwd_model.propagate_differentiable_batched(environment=environment_tiled,
                                                                state=start_states_tiled,
                                                                actions=actions_dict)

    # Run classifier
    state_sequence_length = 2
    accept_probabilities, _ = classifier.check_constraint_batched_tf(environment=environment_tiled,
                                                                     predictions=predictions,
                                                                     actions=actions_dict,
                                                                     state_sequence_length=state_sequence_length,
                                                                     batch_size=n_actions)
    # animate over the sampled actions
    anim = RvizAnimation(scenario=scenario,
                         n_time_steps=n_actions,
                         init_funcs=[init_viz_env],
                         t_funcs=[
                             lambda s, e, t: init_viz_env(s, e),
                             viz_transition_for_model_t({}, fwd_model),
                             ExperimentScenario.plot_accept_probability_t,
                         ],
                         )
    example = {
        'accept_probability': tf.squeeze(accept_probabilities, axis=1),
    }
    example.update(environment)
    example.update(predictions)
    example.update(actions_dict)
    anim.play(example)
Ejemplo n.º 34
0
def squeeze(x, axis):
    '''Removes a 1-dimension from the tensor at index "axis".
    '''
    return tf.squeeze(x, [axis])
Ejemplo n.º 35
0
def get_model(v,
              displacement,
              edges_ss,
              edges_ss_n,
              is_training,
              final_activation,
              bn_decay=None):
    """Generat mesh network

    :param v: variance normalized source vertices
    :param displacement: displacement of each vertex from variance normalized source to variance normalized target
    :param edges_ss: the ij vertix, to compute w_ij, specified for the source vertices
    :param edges_ss_n: number of edges
    :param is_training: need to check what it does, exactly.
    :param final_activation: what activation to put in the final layer that drives w_ij, if range to be limited this hsould be
    sigmoidal.
    :param bn_decay:  need to check what it does, exactly.
    :return: neural network that produces w_ij to be later assambled into a hessian. 
    """

    concat_source_displacement = tf.concat([v, displacement], 2)
    batch_size = v.get_shape()[0].value
    num_point = v.get_shape()[1].value
    input_image = tf.expand_dims(concat_source_displacement, -1)

    net = tf_util.conv2d(input_image,
                         64, [1, 4],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv1',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv2',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv3',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv4',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         1024, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv5',
                         bn_decay=bn_decay)

    # net = tf_util.dropout(net, keep_prob=0.7, is_training=is_training,
    #                       scope='dp1')

    global_feat = tf_util.max_pool2d(net, [num_point, 1],
                                     padding='VALID',
                                     scope='maxpool')

    global_feat_expand = tf.tile(global_feat, [1, edges_ss_n, 1, 1])
    edges_expanded = tf.expand_dims(edges_ss, axis=2)
    concat_feat = tf.concat([edges_expanded, global_feat_expand], 3)

    net = tf_util.conv2d(concat_feat,
                         512, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv6',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         256, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv7',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv8',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv9',
                         bn_decay=bn_decay)
    dim = 2
    net = tf_util.conv2d(net,
                         dim, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         activation_fn=final_activation,
                         is_training=is_training,
                         scope='conv10')

    w_ij = tf.squeeze(net, [2])  # BxNxC

    return w_ij
Ejemplo n.º 36
0
def mesh_get_loss(w_ij,
                  deng_x_at_x0_sub,
                  eng_at_x0_sub,
                  eng_at_xnew_sub,
                  d_n,
                  alpha,
                  epsilon,
                  BATCH=1,
                  nv_sub=12):
    """s    : Source and target patch size on which we test the energy (1-ring) as
               opposed to a slightly larger patch that the networks sees.
               (hessian will be "BATCH x S x S x DATA_DIM", currently data dim = 1);

        w_ij : output of all the weight in hessian. Each w_ij is driven by the same network
               replicated for each edge. So a w_ij is a function of source patch, target
               patch and an edge. There are (S *(S-1))/2 edges in patch of size S,
               leading to all w_ij being a vector of size "BATCH x (S *(S-1))/2 x DATA_DIM"";

        gard : gradient with respect to target of E_{source} @ target.
               Vector of size "BATCH x S x DATA_DIM";

        epsilon : noise added to gradient size  "BATCH x S x DATA_DIM";

        alpha : step size  "BATCH x 1";

        energy_true : Energy value  E_{source} @ (target + alpha grad + epsilon)  "BATCH x 1".

    TODO: currently support DATA_DIM = 1"""

    end_points = {}

    # update hessian with weights where the weight go to the indexes
    # specified by all_indixing_with_batch array.
    wx_ij_flat = tf.reshape(w_ij[:, :, 0], [-1])
    wy_ij_flat = tf.reshape(w_ij[:, :, 1], [-1])

    Hx_final, Hy_final = assemble_h(wx_ij_flat, wy_ij_flat)

    # compute quadratic form
    v = tf.reshape(alpha, (BATCH, 1, 1)) * d_n + epsilon

    vx = tf.reshape(v[:, :, 0], (BATCH, nv_sub, 1))
    vy = tf.reshape(v[:, :, 1], (BATCH, nv_sub, 1))

    # compute quadratic form v^top Q v
    Qx = tf.matmul(a=tf.matmul(a=vx,
                               b=tf.cast(Hx_final, tf.float32),
                               transpose_a=True),
                   b=vx)
    Qx = tf.squeeze(Qx, [2])
    Qy = tf.matmul(a=tf.matmul(a=vy,
                               b=tf.cast(Hy_final, tf.float32),
                               transpose_a=True),
                   b=vy)
    Qy = tf.squeeze(Qy, [2])

    # remove the linear part
    v_flat = tf.reshape(v, [BATCH, -1, 1])
    denergy_dx_flat = tf.reshape(deng_x_at_x0_sub, [BATCH, -1, 1])

    # normalized direction times alpha: v_flat gradient, but gradient is not normalized
    tmp = tf.matmul(a=v_flat, b=denergy_dx_flat, transpose_a=True)
    energy_lin_removed = eng_at_xnew_sub - (eng_at_x0_sub -
                                            tf.squeeze(tmp, [2]))

    energy_dif = energy_lin_removed - Qx - Qy

    end_points['eng_at_xnew_sub'] = eng_at_xnew_sub
    end_points['energy_dif'] = energy_dif
    end_points['Qx'] = Qx
    end_points['Qy'] = Qy
    end_points['energy_lin_removed'] = energy_lin_removed
    end_points['Hx_final'] = Hx_final
    end_points['Hy_final'] = Hy_final

    energy_dif_loss = tf.nn.l2_loss(energy_dif)
    tf.summary.scalar('energy_dif_loss', energy_dif_loss)

    return (energy_dif_loss, end_points)
Ejemplo n.º 37
0
    radius = 1.7
    shift = np.asarray((0.5, 0.1, 0.5)) #shift from voxelization
    center = np.array([0, 0.4, 0]) + shift

    angle = np.pi/2 - i * 2 * np.pi / img_num
    pos = np.array([radius * np.sin(angle), 0.5, radius * np.cos(angle)]) + shift

    mv = pyrr.matrix44.create_look_at(eye=pos, target=center, up=np.array([0, 1, 0])).transpose()

    matrix = np.matmul(p, mv)
    matrix = np.linalg.inv(matrix)

    tf.keras.backend.set_learning_phase(False)
    img = generator([model, matrix])
    img = img * 0.5 + 0.5
    img = tf.squeeze(img)
    img = tf.image.convert_image_dtype(img, dtype=tf.uint8, saturate=True)

    images.append(np.array(img)) # for gif

    if not os.path.isdir(join(log_dir, name)):
        os.mkdir(join(log_dir, name))
    img = tf.image.encode_png(img)
    tf.write_file(join(log_dir, name, str(i).zfill(3)) + ".png", img)

if build_gif:
    imageio.mimsave(join(log_dir, name + ".gif"), images)



Ejemplo n.º 38
0
    def _distance_multihead(self, item_seq, user_latent_factor, item_id):
        # horizontal conv layer
        lengths = [i + 1 for i in range(self.L)]

        out, out_h, out_v = None, None, None

        # item_latent_factor = self.add_timing_signal(item_latent_factor)
        item_latent_factor = tf.nn.embedding_lookup(self.Q, item_seq)
        item_latent_factor_2 = tf.nn.embedding_lookup(self.X, item_seq)

        query = key = self.add_timing_signal(item_latent_factor)

        out = self.multihead_attention(queries=query,
                                       keys=key,
                                       value=item_latent_factor,
                                       reuse=tf.AUTO_REUSE)
        out = tf.reduce_mean(out, 1)

        query_2 = key_2 = out
        query_2 = tf.layers.dense(
            inputs=query_2,
            name="linear_project1",
            units=self.num_factor,
            activation=None,
            use_bias=False,
            kernel_initializer=tf.contrib.layers.xavier_initializer(),
            reuse=tf.AUTO_REUSE,
            kernel_regularizer=tf.contrib.layers.l2_regularizer(
                scale=self.reg_rate))
        key_2 = tf.layers.dense(
            inputs=key_2,
            name="linear_project1",
            units=self.num_factor,
            activation=None,
            use_bias=False,
            kernel_initializer=tf.contrib.layers.xavier_initializer(),
            reuse=tf.AUTO_REUSE,
            kernel_regularizer=tf.contrib.layers.l2_regularizer(
                scale=self.reg_rate))
        # value = tf.layers.dense(inputs= key, name="linear_project", units = seq_len_item, activation = None,  kernel_initializer=tf.random_normal_initializer, reuse=True)
        # b =  tf.Variable(tf.random_normal([seq_len_user], stddev=1))
        logits_2 = tf.matmul(query_2, key_2, transpose_b=True) / np.sqrt(
            self.num_factor)
        weights_2 = tf.nn.softmax(logits_2, name="attention_weights1")
        mask_2 = tf.ones([self.L, self.L])
        mask_2 = tf.matrix_set_diag(mask_2, tf.zeros([self.L]))
        weights_2 = weights_2 * mask_2
        out_2 = tf.reduce_mean(tf.matmul(weights_2, out), 1)

        print("--------------")
        print(np.shape(out))
        print(np.shape(out_2))

        w_items = tf.nn.embedding_lookup(self.X, item_id)
        w_items_2 = tf.nn.embedding_lookup(self.Q, item_id)
        b_items = tf.nn.embedding_lookup(self.b, item_id)
        item_specific_bias = tf.nn.embedding_lookup(self.X, item_id)

        x_tmp = []
        for i in range(np.shape(w_items)[1]):
            x_tmp.append(out)
        x = tf.stack(x_tmp)
        print(np.shape(x))
        print(np.shape(w_items))
        x = tf.transpose(x, [1, 0, 2])

        u_tmp = []
        for i in range(np.shape(w_items)[1]):
            u_tmp.append(user_latent_factor)
        u = tf.stack(u_tmp)
        print(np.shape(u))
        u = tf.transpose(u, [1, 0, 2])

        # res = tf.reduce_sum(tf.multiply(x, w_items), 2) + b_items
        res = 0.2 * tf.reduce_sum(
            tf.square(w_items - u), 2) + 0.8 * tf.reduce_sum(
                tf.square(x - w_items_2),
                2)  # + 0.1 * tf.reduce_sum(tf.square(x - u), 2)

        print(np.shape(res))
        return tf.squeeze(res)
Ejemplo n.º 39
0
 def tf_predict(self, states, internals, update):
     embedding = self.network.apply(x=states, internals=internals, update=update)
     prediction = self.linear.apply(x=embedding)
     return tf.squeeze(input=prediction, axis=1)
Ejemplo n.º 40
0
    def inference(self, inputs, is_training):

        d_out = self.config.d_out
        feature = inputs['features']
        feature = tf.layers.dense(feature, 8, activation=None, name='fc0')
        feature = tf.nn.leaky_relu(
            tf.layers.batch_normalization(feature,
                                          -1,
                                          0.99,
                                          1e-6,
                                          training=is_training))
        feature = tf.expand_dims(feature, axis=2)

        # ###########################Encoder############################
        f_encoder_list = []
        for i in range(self.config.num_layers):
            f_encoder_i = self.dilated_res_block(feature, inputs['xyz'][i],
                                                 inputs['neigh_idx'][i],
                                                 d_out[i],
                                                 'Encoder_layer_' + str(i),
                                                 is_training)
            f_sampled_i = self.random_sample(f_encoder_i, inputs['sub_idx'][i])
            feature = f_sampled_i
            # if i == 0:
            #     self.encoder_features = f_encoder_i
            #     self.encoder_sub_indices = inputs['sub_idx'][i]
            if i == 0:
                f_encoder_list.append(f_encoder_i)
                self.encoder_features.append(f_encoder_i)
            f_encoder_list.append(f_sampled_i)
            self.encoder_features.append(f_sampled_i)
            self.encoder_sub_indices.append(inputs['sub_idx'][i])
        # ###########################Encoder############################

        feature = helper_tf_util.conv2d(
            f_encoder_list[-1], f_encoder_list[-1].get_shape()[3].value,
            [1, 1], 'decoder_0', [1, 1], 'VALID', True, is_training)

        # ###########################Decoder############################
        f_decoder_list = []
        for j in range(self.config.num_layers):
            f_interp_i = self.nearest_interpolation(
                feature, inputs['interp_idx'][-j - 1])
            f_decoder_i = helper_tf_util.conv2d_transpose(
                tf.concat([f_encoder_list[-j - 2], f_interp_i], axis=3),
                f_encoder_list[-j - 2].get_shape()[-1].value, [1, 1],
                'Decoder_layer_' + str(j), [1, 1],
                'VALID',
                bn=True,
                is_training=is_training)
            feature = f_decoder_i
            f_decoder_list.append(f_decoder_i)
            self.decoder_features.append(f_decoder_i)
            self.decoder_sub_indices.append(inputs['interp_idx'][-j - 1])
        # ###########################Decoder############################

        f_layer_fc1 = helper_tf_util.conv2d(f_decoder_list[-1], 64, [1, 1],
                                            'fc1', [1, 1], 'VALID', True,
                                            is_training)
        f_layer_fc2 = helper_tf_util.conv2d(f_layer_fc1, 32, [1, 1], 'fc2',
                                            [1, 1], 'VALID', True, is_training)
        f_layer_drop = helper_tf_util.dropout(f_layer_fc2,
                                              keep_prob=0.5,
                                              is_training=is_training,
                                              scope='dp1')
        f_layer_fc3 = helper_tf_util.conv2d(f_layer_drop,
                                            self.config.num_classes, [1, 1],
                                            'fc', [1, 1],
                                            'VALID',
                                            False,
                                            is_training,
                                            activation_fn=None)
        f_out = tf.squeeze(f_layer_fc3, [2])
        return f_out
def main():
    """Create the model and start the training."""

    start = time.time()
    args = get_arguments()

    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)

    tf.set_random_seed(args.random_seed)

    # Create queue coordinator.
    coord = tf.train.Coordinator()

    # Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReader(
            args.data_dir,
            args.data_list,
            input_size,
            args.random_scale,
            args.random_mirror,
            args.ignore_label,
            coord)
        image_batch, label_batch = reader.dequeue(args.batch_size)

    # Create network.
    net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)

    # Predictions.
    raw_output = net.layers['fc1_voc12']

    restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
    all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
    fc_trainable = [v for v in all_trainable if 'fc' in v.name]
    conv_trainable = [v for v in all_trainable if 'fc' not in v.name]  # lr * 1.0
    fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name]  # lr * 10.0
    fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name]  # lr * 20.0
    assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
    assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))

    # Predictions: ignoring all predictions with labels greater or equal than n_classes
    raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
    label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes,
                               one_hot=False)  # [batch_size, h, w]
    raw_gt = tf.reshape(label_proc, [-1, ])
    indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
    gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
    prediction = tf.gather(raw_prediction, indices)

    # Pixel-wise softmax loss.
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
    l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
    reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)

    # Processed predictions: for visualisation.
    raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ])
    raw_output_up = tf.argmax(raw_output_up, axis=3)
    pred = tf.expand_dims(raw_output_up, dim=3)

    # Define loss and optimisation parameters.
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))

    opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
    opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
    opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)

    grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
    grads_conv = grads[:len(conv_trainable)]
    grads_fc_w = grads[len(conv_trainable): (len(conv_trainable) + len(fc_w_trainable))]
    grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]

    train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
    train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
    train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))

    train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)

    # Set up tf session and initialize variables. 
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()

    sess.run(init)

    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)

    # Load variables if the checkpoint is provided.
    if args.restore_from is not None:
        loader = tf.train.Saver(var_list=restore_var)
        load(loader, sess, args.restore_from)

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)
    losses = []
    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        feed_dict = {step_ph: step}

        if step % args.save_pred_every == 0:
            loss_value, images, labels, preds, _ = sess.run([reduced_loss, image_batch, label_batch, pred, train_op],
                                                            feed_dict=feed_dict)
            save(saver, sess, args.snapshot_dir, step)
        else:
            loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
        duration = time.time() - start_time
        print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
        losses.append(loss_value)
    plotLoss(losses)
    coord.request_stop()
    coord.join(threads)

    end = time.time()
    print("Duration taken to complete the training process is {} seconds.".format(end - start))

    savingParams = []
    savingParams.append(loss_value)
    savingParams.append(end - start)

    # write saved parameters from training
    with open(savingOutputDir, 'w') as f:
        print(savingParams, file=f)
Ejemplo n.º 42
0
    def _distance_self_attention(self,
                                 item_seq,
                                 user_id,
                                 item_id,
                                 isTrain=True):
        # horizontal conv layer
        lengths = [i + 1 for i in range(self.L)]

        out, out_h, out_v = None, None, None
        # print(np.shape(item_seq)[1])

        # item_latent_factor = self.add_timing_signal(item_latent_factor)

        item_latent_factor = tf.nn.embedding_lookup(
            self.Q, item_seq)  #+ self.user_specific_bias

        item_latent_factor_2 = tf.nn.embedding_lookup(self.X, item_seq)

        query = key = value = self.add_timing_signal(item_latent_factor)

        if isTrain:
            query = tf.layers.dense(
                inputs=query,
                name="linear_project",
                units=self.num_factor,
                activation=tf.nn.relu,
                use_bias=False,
                kernel_initializer=tf.contrib.layers.xavier_initializer(),
                reuse=tf.AUTO_REUSE,
                kernel_regularizer=tf.contrib.layers.l2_regularizer(
                    scale=self.reg_rate))
            #query = tf.layers.dropout(query, rate=0.0)
            key = tf.layers.dense(
                inputs=key,
                name="linear_project",
                units=self.num_factor,
                activation=tf.nn.relu,
                use_bias=False,
                kernel_initializer=tf.contrib.layers.xavier_initializer(),
                reuse=tf.AUTO_REUSE,
                kernel_regularizer=tf.contrib.layers.l2_regularizer(
                    scale=self.reg_rate),
            )
            #key = tf.layers.dropout(key, rate=0.3)
        else:
            query = tf.layers.dense(
                inputs=query,
                name="linear_project",
                units=self.num_factor,
                activation=tf.nn.relu,
                use_bias=False,
                kernel_initializer=tf.contrib.layers.xavier_initializer(),
                reuse=tf.AUTO_REUSE,
                kernel_regularizer=tf.contrib.layers.l2_regularizer(
                    scale=self.reg_rate))
            key = tf.layers.dense(
                inputs=key,
                name="linear_project",
                units=self.num_factor,
                activation=tf.nn.relu,
                use_bias=False,
                kernel_initializer=tf.contrib.layers.xavier_initializer(),
                reuse=tf.AUTO_REUSE,
                kernel_regularizer=tf.contrib.layers.l2_regularizer(
                    scale=self.reg_rate))

        logits = tf.matmul(query, key, transpose_b=True) / np.sqrt(
            self.num_factor)

        print(np.shape(logits))
        weights = tf.nn.softmax(logits, dim=-1, name="attention_weights")
        mask = tf.ones([self.L, self.L])
        mask = tf.matrix_set_diag(mask, tf.zeros([self.L]))
        weights = weights * mask

        out = tf.matmul(weights, item_latent_factor)
        self.weights = weights

        print(np.shape(item_latent_factor))
        self.out = tf.reduce_mean(out, 1)
        print(np.shape(self.out))

        w_items = tf.nn.embedding_lookup(self.X, item_id)
        w_items_2 = tf.nn.embedding_lookup(self.Q, item_id)
        w_items_3 = tf.nn.embedding_lookup(
            self.V, user_id)  #tf.nn.embedding_lookup(self.A, item_id)
        self.bias_item = tf.nn.embedding_lookup(self.b, item_id)

        x_tmp = []
        for i in range(np.shape(item_id)[1]):
            x_tmp.append(self.out)
        x = tf.stack(x_tmp)
        print(np.shape(x))
        print(np.shape(w_items))
        x = tf.transpose(x, [1, 0, 2])

        self.user_latent_factor = tf.nn.embedding_lookup(self.P, user_id)

        u_tmp = []
        for i in range(np.shape(item_id)[1]):
            u_tmp.append(self.user_latent_factor)
        u = tf.stack(u_tmp)
        print(np.shape(u))
        u = tf.transpose(u, [1, 0, 2])

        self.user_specific_bias = tf.nn.embedding_lookup(self.V, user_id)
        u_tmp_2 = []
        for i in range(np.shape(item_id)[1]):
            u_tmp_2.append(self.user_specific_bias)
        u_2 = tf.stack(u_tmp_2)
        print(np.shape(u_2))
        u_2 = tf.transpose(u_2, [1, 0, 2])

        self.alpha = 0.2
        if isTrain:
            res = self.alpha * tf.reduce_sum(
                tf.nn.dropout(tf.square(w_items - u), 1),
                2) + (1 - self.alpha) * tf.reduce_sum(
                    tf.nn.dropout(tf.square(x - w_items_2), 1),
                    2)  #+ 0.1 * tf.reduce_sum(tf.square(x - u), 2)
        else:
            res = self.alpha * tf.reduce_sum(tf.square(w_items - u), 2) + (
                1 - self.alpha) * tf.reduce_sum(tf.square(x - w_items_2), 2)

        print(np.shape(res))
        return tf.squeeze(res)
Ejemplo n.º 43
0
        img = sample.reshape(32, 32,3)
        plt.imshow(toimage(img),interpolation='nearest')

    return fig

initializer = tf.contrib.layers.xavier_initializer()
rand_uniform = tf.random_uniform_initializer(-1,1,seed=2)

X = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
N = tf.placeholder(tf.float32, shape=[None, 100])
(x_train, y_train), (x_test, y_test) = load_data()
#x_train = np.concatenate((x_train, x_test), axis=0)
#y_train = np.concatenate((y_train, y_test), axis=0)

x_train = normalize(x_train)
y_train_one_hot = tf.squeeze(tf.one_hot(y_train, 10),axis=1)


theta_G =[]
def xavier_init(size):
    in_dim = size[0]
    xavier_stddev = 1. / tf.sqrt(in_dim / 2.)
    return tf.random_normal(shape=size, stddev=xavier_stddev)

def autoencoder(x):
    input_shape=[None, 32, 32, 3]
    n_filters=[3, 128, 256, 512]
    filter_sizes=[5, 5, 5, 5]
    
    if len(x.get_shape()) == 3:
        x_dim = np.sqrt(x.get_shape().as_list()[1])
Ejemplo n.º 44
0
def triangulate(startpoints, endpoints, weights, name="ray_triangulate"):
  """Triangulates 3d points by miminizing the sum of squared distances to rays.

  The rays are defined by their start points and endpoints. At least two rays
  are required to triangulate any given point. Contrary to the standard
  reprojection-error metric, the sum of squared distances to rays can be
  minimized in a closed form.

  Note:
    In the following, A1 to An are optional batch dimensions.

  Args:
    startpoints: A tensor of ray start points with shape `[A1, ..., An, V, 3]`,
      the number of rays V around which the solution points live should be
      greater or equal to 2, otherwise triangulation is impossible.
    endpoints: A tensor of ray endpoints with shape `[A1, ..., An, V, 3]`, the
      number of rays V around which the solution points live should be greater
      or equal to 2, otherwise triangulation is impossible. The `endpoints`
      tensor should have the same shape as the `startpoints` tensor.
    weights: A tensor of ray weights (certainties) with shape `[A1, ..., An,
      V]`. Weights should have all positive entries. Weight should have at least
      two non-zero entries for each point (at least two rays should have
      certainties > 0).
    name: A name for this op. The default value of None means "ray_triangulate".

  Returns:
    A tensor of triangulated points with shape `[A1, ..., An, 3]`.

  Raises:
    ValueError: If the shape of the arguments is not supported.
  """
  with tf.name_scope(name):
    startpoints = tf.convert_to_tensor(value=startpoints)
    endpoints = tf.convert_to_tensor(value=endpoints)
    weights = tf.convert_to_tensor(value=weights)

    shape.check_static(
        tensor=startpoints,
        tensor_name="startpoints",
        has_rank_greater_than=1,
        has_dim_equals=(-1, 3),
        has_dim_greater_than=(-2, 1))
    shape.check_static(
        tensor=endpoints,
        tensor_name="endpoints",
        has_rank_greater_than=1,
        has_dim_equals=(-1, 3),
        has_dim_greater_than=(-2, 1))
    shape.compare_batch_dimensions(
        tensors=(startpoints, endpoints, weights),
        last_axes=(-2, -2, -1),
        broadcast_compatible=False)
    weights = asserts.assert_all_above(weights, 0.0, open_bound=False)
    weights = asserts.assert_at_least_k_non_zero_entries(weights, k=2)

    left_hand_side_list = []
    right_hand_side_list = []
    # TODO(b/130892100): Replace the inefficient for loop and add comments here.
    for ray_id in range(weights.shape[-1]):
      weights_single_ray = weights[..., ray_id]
      startpoints_single_ray = startpoints[..., ray_id, :]
      endpoints_singleview = endpoints[..., ray_id, :]
      ray = endpoints_singleview - startpoints_single_ray
      ray = tf.nn.l2_normalize(ray, axis=-1)
      ray_x, ray_y, ray_z = tf.unstack(ray, axis=-1)
      zeros = tf.zeros_like(ray_x)
      cross_product_matrix = tf.stack(
          (zeros, -ray_z, ray_y, ray_z, zeros, -ray_x, -ray_y, ray_x, zeros),
          axis=-1)
      cross_product_matrix_shape = tf.concat(
          (tf.shape(input=cross_product_matrix)[:-1], (3, 3)), axis=-1)
      cross_product_matrix = tf.reshape(
          cross_product_matrix, shape=cross_product_matrix_shape)
      weights_single_ray = tf.expand_dims(weights_single_ray, axis=-1)
      weights_single_ray = tf.expand_dims(weights_single_ray, axis=-1)
      left_hand_side = weights_single_ray * cross_product_matrix
      left_hand_side_list.append(left_hand_side)
      dot_product = tf.matmul(cross_product_matrix,
                              tf.expand_dims(startpoints_single_ray, axis=-1))
      right_hand_side = weights_single_ray * dot_product
      right_hand_side_list.append(right_hand_side)
    left_hand_side_multi_rays = tf.concat(left_hand_side_list, axis=-2)
    right_hand_side_multi_rays = tf.concat(right_hand_side_list, axis=-2)
    points = tf.linalg.lstsq(left_hand_side_multi_rays,
                             right_hand_side_multi_rays)
    points = tf.squeeze(points, axis=-1)

    return points
Ejemplo n.º 45
0
    def train(self):
        Weight = tf.Variable(
            tf.truncated_normal([self.corpus.n_words, const.EMBEDDING_SIZE],
                                stddev=1.0 / math.sqrt(const.EMBEDDING_SIZE)))
        bias = tf.Variable(tf.random_normal([self.corpus.n_words]))

        inputs = tf.placeholder(tf.int32, [const.BATCH_SIZE, 1])
        targets = tf.placeholder(tf.int32, [const.BATCH_SIZE, 1])
        vocabs = tf.placeholder(tf.int32,
                                [const.BATCH_SIZE, self.corpus.n_words])

        embed_weight_v = tf.Variable(
            tf.random_normal([self.corpus.n_words, const.EMBEDDING_SIZE], -1.0,
                             1.0))
        embed_weight_u = tf.Variable(
            tf.random_normal([self.corpus.n_words, const.EMBEDDING_SIZE], -1.0,
                             1.0))
        embed_weight_actual = tf.Variable(
            tf.random_normal([self.corpus.n_words, const.EMBEDDING_SIZE], -1.0,
                             1.0))
        embed_v = tf.nn.embedding_lookup(embed_weight_v, inputs)
        embed_u = tf.nn.embedding_lookup(embed_weight_u, targets)
        embed_actual = tf.nn.embedding_lookup(embed_weight_actual, vocabs)
        '''
		print(embed_u.shape)
		print(embed_v.shape)
		print(embed_actual.shape)
		exit()
		'''
        embed_v_trans = tf.transpose(embed_v, [0, 2, 1])

        #print(embed_v_trans.shape)
        scores = tf.squeeze(tf.matmul(embed_u, embed_v_trans),
                            [2])  # batch_size x 1
        norm_scores = tf.squeeze(tf.matmul(embed_actual, embed_v_trans),
                                 [2])  # batch_size x input_size

        softmax = tf.exp(scores) / tf.reduce_sum(tf.exp(norm_scores), 1)
        softmax = tf.expand_dims(softmax, 1)
        nll_loss = -tf.reduce_mean(
            tf.log(tf.clip_by_value(softmax, 1e-10, 1.0)))

        optimizer = tf.train.AdamOptimizer(
            learning_rate=const.LR_RATE).minimize(nll_loss)

        saver = tf.train.Saver()

        losses = []
        with tf.Session() as sess:
            tf.global_variables_initializer().run()
            for epoch in range(const.EPOCH):
                for i, batch in enumerate(
                        self.corpus.batch_data(const.BATCH_SIZE)):
                    _inputs, _targets = zip(*batch)  # unzip

                    _inputs = np.hstack(_inputs)  # (2, )
                    _inputs = _inputs.reshape(_inputs.shape[0], 1)
                    _targets = np.vstack(_targets)  # (2, 1)

                    vocab = self.corpus.var_sentence(self.corpus.vocab)
                    _vocabs = []
                    [_vocabs.append(vocab) for x in range(inputs.shape[0])]
                    _vocabs = np.array(_vocabs)

                    _, _loss = sess.run([optimizer, nll_loss],
                                        feed_dict={
                                            inputs: _inputs,
                                            targets: _targets,
                                            vocabs: _vocabs
                                        })
                    losses.append(_loss)
                    if i % 500:
                        print('i, ', i, 'loss', _loss)

                if epoch % 10 == 0:
                    print('epoch, ', epoch, 'mean loss', np.mean(losses))
                    losses = []

            # save model
            saver.save(sess, const.MODEL_PATH)
Ejemplo n.º 46
0
    def sub_func(example_proto):
        features = {}
        label = {}
        for col in used_fea.categorical_columns:
            if col not in ['appId_list_encoded']:
                features[col] = tf.FixedLenFeature((1),
                                                   tf.int64,
                                                   default_value=0)
        for col in used_fea.numerical_columns:
            features[col] = tf.FixedLenFeature((1),
                                               tf.float32,
                                               default_value=0.1)
        features['appId_list_encoded'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_appId_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_duration_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_times_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_use_date_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['all_activedApp_cate_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_appId_duration_list'] = tf.VarLenFeature(
            dtype=tf.int64)
        features['usage_appId_times_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_appId_mean_dura_list'] = tf.VarLenFeature(
            dtype=tf.int64)
        features['usage_appId_full_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_duration_full_list'] = tf.VarLenFeature(dtype=tf.int64)
        features['usage_time_full_list'] = tf.VarLenFeature(dtype=tf.int64)

        parsed_features = tf.parse_single_example(example_proto, features)
        parsed_features['appId_list_encoded'] = tf.sparse_tensor_to_dense(
            parsed_features['appId_list_encoded'])
        parsed_features['usage_appId_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_appId_list'])
        parsed_features['usage_duration_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_duration_list'])
        parsed_features['usage_times_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_times_list'])
        parsed_features['usage_use_date_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_use_date_list'])
        parsed_features[
            'all_activedApp_cate_list'] = tf.sparse_tensor_to_dense(
                parsed_features['all_activedApp_cate_list'])

        parsed_features[
            'usage_appId_duration_list'] = tf.sparse_tensor_to_dense(
                parsed_features['usage_appId_duration_list'])
        parsed_features['usage_appId_times_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_appId_times_list'])
        parsed_features[
            'usage_appId_mean_dura_list'] = tf.sparse_tensor_to_dense(
                parsed_features['usage_appId_mean_dura_list'])

        parsed_features['usage_appId_full_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_appId_full_list'])
        parsed_features[
            'usage_duration_full_list'] = tf.sparse_tensor_to_dense(
                parsed_features['usage_duration_full_list'])
        parsed_features['usage_time_full_list'] = tf.sparse_tensor_to_dense(
            parsed_features['usage_time_full_list'])

        label['age_group'] = tf.FixedLenFeature((1),
                                                tf.float32,
                                                default_value=1)
        parsed_label = tf.parse_single_example(example_proto, label)
        parsed_label['age_group'] = tf.cast(parsed_label['age_group'],
                                            tf.int64)
        label = tf.cast(tf.one_hot(parsed_label['age_group'] - 1, 6, 1, 0),
                        tf.float32)
        label = tf.squeeze(label, squeeze_dims=[0])
        return parsed_features, label
Ejemplo n.º 47
0
    def sample(self,
               ztype='prior',
               num_samples=1,
               seed=None,
               linear_predictors=None,
               checkpoint_file=None,
               z_in=None,
               y_in=None,
               gmm_params=None,
               mark_probs=None):
        """
        Generate samples from prior/posterior and model

        Args:
            ztype (str): distribution used for latent state samples
                'prior' | 'posterior'
            num_samples (int, optional)
            seed (int, optional): random seed for reproducibly generating
                random samples
            linear_predictors (list of np arrays):
                1 x num_time_pts x dim_pred;
                there will be num_samples random samples for this one example
                of each linear predictor
            checkpoint_file (str, optional): checkpoint file specifying model
                from which to generate samples; if `None`, will then look for a
                checkpoint file created upon model initialization

        Returns:
            num_samples x num_time_pts x dim_obs numpy array: y
            num_samples x num_time_pts x dim_latent numpy array: z

        Raises:
            ValueError: for incorrect `ztype` values

        """

        self._check_graph()

        # intialize session
        with tf.Session(graph=self.graph, config=self.sess_config) as sess:
            self.restore_model(sess, checkpoint_file=checkpoint_file)
            #            print(self.gen_net.networks[0].layers[0].kernel.eval(sess))
            if z_in is not None:
                # this only works in one specific case
                ztmp = tf.convert_to_tensor(z_in, dtype=self.dtype)
                ymean = self.gen_net.networks[0].apply_network(ztmp)
                if mark_probs is not None:
                    #                    if mark_probs is None:
                    #                        raise ValueError('Must provide mark probabilities to evaluate this model')
                    F = tf.expand_dims(ymean, axis=2)
                    W = tf.convert_to_tensor(mark_probs, dtype=self.dtype)
                    ymean = tf.squeeze(tf.matmul(F, W))

                y_ = tf.random_poisson(lam=ymean,
                                       shape=[1],
                                       dtype=tf.float32,
                                       seed=seed)
                y = np.squeeze(y_.eval())
                z = z_in
#                [y, z] = sess.run([y_, ztmp], feed_dict=feed_dict)

            elif ztype is 'prior':
                y, z = self.gen_net.sample(sess, num_samples, seed,
                                           linear_predictors, mark_probs)

            elif ztype is 'posterior':
                if y_in is None:
                    raise ValueError(
                        '`y_in` required to sample from posterior')
#                 TODO: needs input to inference network as well
                z = self.inf_net.sample(sess,
                                        y_in,
                                        mark_probs=mark_probs,
                                        seed=seed)
                ztmp = tf.convert_to_tensor(z, dtype=self.dtype)
                ymean = self.gen_net.networks[0].apply_network(ztmp)
                if mark_probs is not None:
                    #                    if mark_probs is None:
                    #                        raise ValueError('Must provide mark probabilities to evaluate this model')
                    F = tf.transpose(ymean, (0, 2, 1, 3))
                    W = tf.convert_to_tensor(mark_probs, dtype=self.dtype)
                    ymean = tf.transpose(tf.matmul(F, W), (0, 2, 1, 3))

                y_ = tf.random_poisson(lam=ymean,
                                       shape=[1],
                                       dtype=tf.float32,
                                       seed=seed)
                y = np.squeeze(y_.eval())
            else:
                raise ValueError('Invalid string "%s" for ztype argument')

            # simulate the marks -- takes a long time
            if gmm_params is not None:
                ztmp = tf.convert_to_tensor(z, dtype=self.dtype)
                f_k = self.gen_net.networks[0].apply_network(ztmp)
                f_k = f_k.eval()
                f_k = np.reshape(f_k, (-1, f_k.shape[2]))
                y_flat = np.reshape(y, (-1, self.dim_obs[0]))

                mrk = []
                ii = 0
                for i in range(self.dim_obs[0]):
                    mus = gmm_params[i][0]
                    sigs = gmm_params[i][1]
                    pi = gmm_params[i][2]

                    dim_marks = mus.shape[0]

                    k = pi.shape[0]
                    f_ik = f_k[:, ii:(ii + k)]
                    ii += k

                    log_pi_hat = np.log(f_ik) + np.log(pi +
                                                       1e-16)[np.newaxis, :]
                    c = log_pi_hat.max(1)[:, np.newaxis]
                    log_norm = c + np.log(
                        np.sum(np.exp(log_pi_hat - c), axis=1))[:, np.newaxis]
                    log_pi_hat -= log_norm
                    pi_hat = np.exp(log_pi_hat)

                    temp_mrk = np.zeros((0, dim_marks))
                    ind = np.zeros(0)
                    for t in np.where(y_flat[:, i] > 0)[0]:
                        Nt = int(y_flat[t, i])
                        zeta = rnd.multinomial(1, pi_hat[t, :],
                                               size=Nt).argmax(1)
                        mu = mus[:, zeta]
                        C = sigs[:, :, zeta]

                        m_t = np.array([rnd.multivariate_normal(mu[:,jj],C[:,:,jj]) \
                                        for jj in range(Nt)], dtype = np.float32)
                        temp_mrk = np.append(temp_mrk, m_t, axis=0)
                        ind = np.append(ind, np.ones(Nt) * int(t))

                    mrk.append([temp_mrk, ind])
            else:
                mrk = None

        return y, z, mrk
    def computemodel(self, is_resnet=False, is_forcepos=False):
        ''' Perform Multiple Scattering here
        1.) Multiple Scattering is perfomed by slice-wise propagation the E-Field throught the sample
        2.) Each Field has to be backprojected to the BFP
        3.) Last step is to ceate a focus-stack and sum over all angles
 
        This is done for all illumination angles (coming from illumination NA
        simultaneasly)'''
        self.is_resnet = is_resnet
        self.is_forcepos = is_forcepos

        print("Buildup Q-PHASE Model ")
        ###### make sure, that the first dimension is "batch"-size; in this case it is the illumination number
        # @beniroquai It's common to have to batch dimensions first and not last.!!!!!
        # the following loop propagates the field sequentially to all different Z-planes

        ## propagate the field through the entire object for all angles simultaneously
        A_prop = np.transpose(
            self.A_input,
            [3, 0, 1, 2
             ])  # ??????? what the hack is happening with transpose?!

        if (self.is_tomo):
            print('Experimentally using the tomographic scheme!')
            A_prop = np.conj(A_prop)

        myprop = np.exp(1j * self.dphi) * (self.dphi > 0)
        # excludes the near field components in each step
        myprop = tf_helper.repmat4d(np.fft.fftshift(myprop), self.Nc)
        myprop = np.transpose(
            myprop, [3, 0, 1, 2
                     ])  # ??????? what the hack is happening with transpose?!

        print('--------> ATTENTION: I added a pi factor - is this correct?!')
        RefrEffect = np.squeeze(1j * np.pi * self.dz * self.k0 * self.RefrCos)
        # Precalculate the oblique effect on OPD to speed it up

        # for now orientate the dimensions as (alpha_illu, x, y, z) - because tensorflow takes the first dimension as batch size
        with tf.name_scope('Variable_assignment'):
            self.TF_A_input = tf.constant(A_prop, dtype=tf.complex64)
            self.TF_RefrEffect = tf.reshape(
                tf.constant(RefrEffect, dtype=tf.complex64), [self.Nc, 1, 1])
            self.TF_myprop = tf.constant(np.squeeze(myprop),
                                         dtype=tf.complex64)
            self.TF_Po = tf.cast(tf.constant(self.Po), tf.complex64)
            self.TF_Zernikes = tf.constant(self.myzernikes, dtype=tf.float32)
            self.TF_myAllSlicePropagator = tf.constant(
                self.myAllSlicePropagator, dtype=tf.complex64)

            # Only update those Factors which are really necesarry (e.g. Defocus is not very likely!)
            self.TF_zernikefactors = tf.Variable(self.zernikefactors,
                                                 dtype=tf.float32,
                                                 name='var_zernikes')
            #indexes = tf.constant([[4], [5], [6], [7], [8], [9]])
            indexes = tf.cast(tf.where(tf.constant(self.zernikemask) > 0),
                              tf.int32)
            updates = tf.gather_nd(self.TF_zernikefactors, indexes)
            # Take slice
            # Build tensor with "filtered" gradient
            part_X = tf.scatter_nd(indexes, updates,
                                   tf.shape(self.TF_zernikefactors))
            self.TF_zernikefactors_filtered = part_X + tf.stop_gradient(
                -part_X + self.TF_zernikefactors)

        # TODO: Introduce the averraged RI along Z - MWeigert
        self.TF_A_prop = tf.squeeze(self.TF_A_input)
        self.U_z_list = []

        # Initiliaze memory
        self.allSumAmp = 0
        self.TF_allSumAmp = tf.zeros([self.mysize[0], self.Nx, self.Ny],
                                     dtype=tf.complex64)
        ''' Eventually add a RESNET-layer between RI and Microscope to correct for model discrepancy?'''
        if (self.is_resnet):
            with tf.variable_scope('res_real', reuse=False):
                TF_real_3D = self.residual_block(
                    tf.expand_dims(tf.expand_dims(self.TF_obj, 3), 0), 1, True)
                TF_real_3D = tf.squeeze(TF_real_3D)
            with tf.variable_scope('res_imag', reuse=False):
                TF_imag_3D = self.residual_block(
                    tf.expand_dims(tf.expand_dims(self.TF_obj_absorption, 3),
                                   0), 1, True)
                TF_imag_3D = tf.squeeze(TF_imag_3D)
        else:
            TF_real_3D = self.TF_obj
            TF_imag_3D = self.TF_obj_absorption

        # wrapper for force-positivity on the RI-instead of penalizing it
        if (self.is_forcepos):
            print('----> ATTENTION: We add the PreMonotonicPos')
            TF_real_3D = tf_reg.PreMonotonicPos(TF_real_3D)
            TF_imag_3D = tf_reg.PreMonotonicPos(TF_imag_3D)

        TF_A_prop_illu_list = []
        # simulate multiple scattering through object
        with tf.name_scope('Fwd_Propagate'):
            #print('---------ATTENTION: We are inverting the RI!')

            # First Iterate over all illumination angles
            for pillu in range(0, self.Nc):
                # Second iterate over all z-slices
                TF_A_prop_z_tmp = self.TF_A_prop[pillu, :, :]
                for pz in range(0, self.mysize[0]):
                    if (self.is_padding):
                        tf_paddings = tf.constant([[
                            self.mysize_old[1] // 2, self.mysize_old[1] // 2
                        ], [self.mysize_old[2] // 2, self.mysize_old[2] // 2]])
                        TF_real = tf.pad(TF_real_3D[-pz, :, :],
                                         tf_paddings,
                                         mode='CONSTANT',
                                         name='TF_obj_real_pad')
                        TF_imag = tf.pad(TF_imag_3D[-pz, :, :],
                                         tf_paddings,
                                         mode='CONSTANT',
                                         name='TF_obj_imag_pad')
                    else:
                        TF_real = (TF_real_3D[-pz, :, :])
                        TF_imag = (TF_imag_3D[-pz, :, :])

                    self.TF_f = tf.exp(self.TF_RefrEffect[pillu, :, :] *
                                       tf.complex(TF_real, TF_imag))
                    #self.TF_A_prop = tf.Print(self.TF_A_prop, [self.tf_iterator], 'Prpagation step: ')
                    with tf.name_scope('Refract'):
                        # beware the "i" is in TF_RefrEffect already!
                        TF_A_prop_z_tmp = TF_A_prop_z_tmp * self.TF_f  # refraction step

                    with tf.name_scope('Propagate'):
                        TF_A_prop_z_tmp = tf.ifft2d(
                            tf.fft2d(TF_A_prop_z_tmp) *
                            self.TF_myprop[pillu, :, :])  # diffraction step

                TF_A_prop_illu_list.append(TF_A_prop_z_tmp)
            self.TF_A_prop = tf.stack(TF_A_prop_illu_list)

        # in a final step limit this to the detection NA:
        self.TF_Po_aberr = tf.exp(1j * tf.cast(
            tf.reduce_sum(self.TF_zernikefactors_filtered * self.TF_Zernikes,
                          axis=2), tf.complex64)) * self.TF_Po
        self.TF_A_prop = tf.ifft2d(
            tf.fft2d(self.TF_A_prop) * self.TF_Po * self.TF_Po_aberr)

        # Experimenting with pseudo tomographic data?
        if self.is_tomo:
            print('Only Experimental! Tomographic data?!')
            # Bring the slice back to focus - does this make any sense?!
            print('----------> Bringing field back to focus')
            TF_centerprop = tf.exp(
                -1j *
                tf.cast(self.Nz / 2 * tf.angle(self.TF_myprop), tf.complex64))
            self.TF_A_prop = tf.ifft2d(
                tf.fft2d(self.TF_A_prop) * TF_centerprop)  # diffraction step
            return self.TF_A_prop

        # create Z-Stack by backpropagating Information in BFP to Z-Position
        self.kzcoord = np.reshape(self.kz[self.Ic > 0], [1, 1, 1, self.Nc])

        # self.mid3D = ([np.int(np.ceil(self.A_input.shape[0] / 2) + 1), np.int(np.ceil(self.A_input.shape[1] / 2) + 1), np.int(np.ceil(self.mysize[0] / 2) + 1)])
        self.mid3D = ([
            np.int(self.mysize[0] // 2),
            np.int(self.A_input.shape[0] // 2),
            np.int(self.A_input.shape[1] // 2)
        ])

        with tf.name_scope('Back_Propagate'):
            print('----------> ATTENTION: PHASE SCRAMBLING!')
            for pillu in range(0, self.Nc):
                TF_allAmp_list = []
                #    fprintf('BackpropaAngle no: #d\n',pillu);
                OneAmp = tf.expand_dims(self.TF_A_prop[pillu, :, :], 0)
                # Fancy backpropagation assuming what would be measured if the sample was moved under oblique illumination:
                # The trick is: First use conceptually the normal way
                # and then apply the XYZ shift using the Fourier shift theorem (corresponds to physically shifting the object volume, scattered field stays the same):
                self.TF_AdjustKXY = tf.squeeze(
                    tf.conj(self.TF_A_input[pillu, :, :, ])
                )  # Maybe a bit of a dirty hack, but we first need to shift the zero coordinate to the center
                self.TF_AdjustKZ = tf.cast(
                    tf.transpose(
                        np.exp(2 * np.pi * 1j * self.dz * np.reshape(
                            np.arange(
                                0, self.mysize[0], 1
                            ),  # We want to start from first Z-slice then go to last which faces the objective lens
                            [1, 1, self.mysize[0]]) *
                               self.kzcoord[:, :, :, pillu]),
                        [2, 1, 0]),
                    tf.complex64)

                for pz in range(0, self.Nz):
                    with tf.name_scope('Back_Propagate_Step'):
                        TF_allAmp_list.append(
                            tf.squeeze(
                                tf.ifft2d(
                                    tf.fft2d(OneAmp) *
                                    self.TF_myAllSlicePropagator[pz, :, :])) *
                            self.TF_AdjustKZ[pz, :, :] *
                            self.TF_AdjustKXY[:, :]
                        )  # * (TF_AdjustKZ);  # 2x bfxfun.  Propagates a single amplitude pattern back to the whole stack
                        #tf_global_phase = tf.cast(tf.angle(self.TF_allAmp[self.mid3D[0],self.mid3D[1],self.mid3D[2]]), tf.complex64)
                        #tf_global_phase = tf.cast(np.random.randn(1)*np.pi,tf.complex64)
                        #self.TF_allAmp = self.TF_allAmp * tf.exp(1j*tf_global_phase) # Global Phases need to be adjusted at this step!  Use the zero frequency

                TF_allAmp = tf.stack(TF_allAmp_list)

                with tf.name_scope('Adjust_Global_Phase'):
                    self.TF_allAmp_3dft = tf.fft3d(
                        tf.expand_dims(TF_allAmp, axis=0))
                    tf_global_phase = tf.angle(
                        self.TF_allAmp_3dft[0, 0, 0, 0]
                    )  #tf.angle(self.TF_allAmp_3dft[0, self.mid3D[2], self.mid3D[1], self.mid3D[0]])
                    tf_global_phase = tf.cast(tf_global_phase, tf.complex64)

                    TF_allAmp = TF_allAmp * tf.exp(-1j * tf_global_phase)
                    # Global Phases need to be adjusted at this step!  Use the zero frequency
                    #print('Global phase: '+str(tf.exp(1j*tf.cast(tf.angle(self.TF_allAmp[self.mid3D[0],self.mid3D[1],self.mid3D[2]]), tf.complex64).eval()))

                with tf.name_scope(
                        'Sum_Amps'
                ):  # Normalize amplitude by condenser intensity
                    self.TF_allSumAmp += TF_allAmp
                    # Superpose the Amplitudes

                    # print('Current illumination angle # is: ' + str(pillu))

        # Normalize the image such that the values do not depend on the fineness of
        # the source grid.
        self.TF_allSumAmp = self.TF_allSumAmp / self.Nc  #/tf.cast(tf.reduce_max(tf.abs(self.TF_allSumAmp)), tf.complex64)
        # Following is the normalization according to Martin's book. It ensures
        # that a transparent specimen is imaged with unit intensity.
        # normfactor=abs(Po).^2.*abs(Ic); We do not use it, because it leads to
        # divide by zero for dark-field system. Instead, through normalizations
        # perfomed above, we ensure that image of a point under matched
        # illumination is unity. The brightness of all the other configurations is
        # relative to this benchmark.
        #

        # negate padding
        if self.is_padding:
            self.TF_allSumAmp = self.TF_allSumAmp[:, self.Nx // 2 -
                                                  self.Nx // 4:self.Nx // 2 +
                                                  self.Nx // 4, self.Ny // 2 -
                                                  self.Ny // 4:self.Ny // 2 +
                                                  self.Ny // 4]

        return self.TF_allSumAmp
Ejemplo n.º 49
0
def inference(batch_placeholders, similarity_placeholder, init_word_embeds,
              word_to_num, num_to_word):
    print("Begin inference:")
    print("Creating variables")

    E = tf.Variable(init_word_embeds, dtype=tf.float32)
    W = tf.Variable(
        tf.random_uniform(
            [params.lstm_size, params.lstm_size, params.slice_size],
            minval=-1.0 / params.lstm_size,
            maxval=1.0 / params.lstm_size,
            name='W'))
    V = tf.Variable(
        tf.random_uniform([params.slice_size, 2 * params.lstm_size],
                          minval=-1.0 / (2 * params.lstm_size),
                          maxval=1.0 / (2 * params.lstm_size)))
    b = tf.Variable(tf.zeros([1, params.slice_size]), name='b')
    U = tf.Variable(
        tf.random_uniform([1, params.slice_size],
                          minval=-1.0 / params.slice_size,
                          maxval=1.0 / params.slice_size))
    lstm = createLSTM(params.lstm_size)
    print("Calcing sentences2vec")
    question_vec, pos_answer_vec, neg1, neg2, neg3 = tf.split(
        1, params.corrupt_size + 2, batch_placeholders)
    #scr_pos_answer, scr_neg1 , scr_neg2 , scr_neg3 = tf.split(1, params.corrupt_size+1,similarity_placeholder)
    #similarity_scores = tf.cast(similarity_placeholder, tf.float32)
    question_vec = tf.squeeze(question_vec)
    pos_answer_vec = tf.squeeze(pos_answer_vec)
    neg1 = tf.squeeze(neg1)
    neg2 = tf.squeeze(neg2)
    neg3 = tf.squeeze(neg3)
    #scr_pos_answer = tf.squeeze(scr_pos_answer)
    #scr_neg1 = tf.squeeze(scr_neg1)
    #scr_neg2 = tf.squeeze(scr_neg2)
    #scr_neg3 = tf.squeeze(scr_neg3)
    #question_vec = tf.reduce_mean(tf.gather(E,question_vec),1)
    question_vec = train_sentence2vectorLSTM(lstm, E, question_vec, False)
    pos_answer_vec = train_sentence2vectorLSTM(lstm, E, pos_answer_vec, True)
    neg1 = train_sentence2vectorLSTM(lstm, E, neg1, True)
    neg2 = train_sentence2vectorLSTM(lstm, E, neg2, True)
    neg3 = train_sentence2vectorLSTM(lstm, E, neg3, True)

    #new_p = tf.zeros([pparams.lstm_size+1])
    #pos_answer_vec = tf.reshape(pos_answer_vec, [-1])
    #print scr_pos_answer.get_shape
    #pos_answer_vec = tf.concat(1,[pos_answer_vec,scr_pos_answer])
    #neg1 = tf.concat(1,[neg1,scr_neg1])
    #neg2 = tf.concat(1,[neg2,scr_neg2])
    #neg3 = tf.concat(1,[neg3,scr_neg3])
    #pos_answer_vec = tf.reduce_mean(tf.gather(E, pos_answer_vec), 1)
    #neg1 = tf.reduce_mean(tf.gather(E, neg1), 1)
    #neg2 = tf.reduce_mean(tf.gather(E, neg2), 1)
    #neg3 = tf.reduce_mean(tf.gather(E, neg3), 1)

    tensors = []
    for i in range(params.slice_size):
        tensor = tf.reduce_sum(
            pos_answer_vec * tf.matmul(question_vec, W[:, :, i]), 1)
        tensors.append(tensor)

    score_pos = tf.pack(tensors)
    vec_concat = tf.transpose(
        tf.matmul(V, tf.transpose(tf.concat(1,
                                            [question_vec, pos_answer_vec]))))
    score_pos = tf.matmul(tf.nn.relu(tf.transpose(score_pos) + vec_concat + b),
                          tf.transpose(U))

    negative = []
    for i in [neg1, neg2, neg3]:
        tensors = []
        for j in range(params.slice_size):
            tensor = tf.reduce_sum(i * tf.matmul(question_vec, W[:, :, j]), 1)
            tensors.append(tensor)

        score_neg = tf.pack(tensors)
        vec_concat = tf.transpose(
            tf.matmul(V, tf.transpose(tf.concat(1, [question_vec, i]))))
        score_neg = tf.matmul(
            tf.nn.relu(tf.transpose(score_neg) + vec_concat + b),
            tf.transpose(U))
        negative.append(score_neg)

    return [score_pos, negative[0], negative[1], negative[2]]
Ejemplo n.º 50
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='data/',
                        help='data directory containing input.txt')
    parser.add_argument('--batch_size', type=int, default=120,
                        help='minibatch size')
    parser.add_argument('--win_size', type=int, default=5,
                        help='context sequence length')
    parser.add_argument('--hidden_num', type=int, default=100,
                        help='number of hidden layers')
    parser.add_argument('--word_dim', type=int, default=300,
                        help='number of word embedding')
    parser.add_argument('--num_epochs', type=int, default=3,
                        help='number of epochs')
    parser.add_argument('--grad_clip', type=float, default=10.,
                        help='clip gradients at this value')

    args = parser.parse_args()
    args_msg = '\n'.join([ arg + ': ' + str(getattr(args, arg)) for arg in vars(args)])


    data_loader = TextLoader(args.data_dir, args.batch_size, args.win_size)
    args.vocab_size = data_loader.vocab_size

    graph = tf.Graph()
    with graph.as_default():
        input_data = tf.placeholder(tf.int64, [args.batch_size, args.win_size])
        targets = tf.placeholder(tf.int64, [args.batch_size, 1])

        with tf.variable_scope('nnlm' + 'embedding'):
            embeddings = tf.Variable(tf.random_uniform([args.vocab_size, args.word_dim], -1.0, 1.0))
            embeddings = tf.nn.l2_normalize(embeddings, 1)

        with tf.variable_scope('nnlm' + 'weight'):
            weight_h = tf.Variable(tf.truncated_normal([args.win_size * args.word_dim, args.hidden_num],
                                                       stddev=1.0 / math.sqrt(args.hidden_num)))
            softmax_w = tf.Variable(tf.truncated_normal([args.win_size * args.word_dim, args.vocab_size],
                                                        stddev=1.0 / math.sqrt(args.win_size * args.word_dim)))
            softmax_u = tf.Variable(tf.truncated_normal([args.hidden_num, args.vocab_size],
                                                        stddev=1.0 / math.sqrt(args.hidden_num)))

            b_1 = tf.Variable(tf.random_normal([args.hidden_num]))
            b_2 = tf.Variable(tf.random_normal([args.vocab_size]))

        def infer_output(input_data):
            """
            hidden = tanh(x * H + b_1)
            output = softmax(x * W + hidden * U + b_2)
            """
            input_data_emb = tf.nn.embedding_lookup(embeddings, input_data)
            input_data_emb = tf.reshape(input_data_emb, [-1, args.win_size * args.word_dim])
            hidden = tf.tanh(tf.matmul(input_data_emb, weight_h)) + b_1
            hidden_output = tf.matmul(hidden, softmax_u) + tf.matmul(input_data_emb, softmax_w) + b_2
            output = tf.nn.softmax(hidden_output)
            return output

        outputs = infer_output(input_data)
        one_hot_targets = tf.one_hot(tf.squeeze(targets), args.vocab_size, 1.0, 0.0)
        loss = -tf.reduce_mean(tf.reduce_sum(tf.log(outputs) * one_hot_targets, 1))
        # Clip grad.
        optimizer = tf.train.AdagradOptimizer(0.1)
        gvs = optimizer.compute_gradients(loss)
        capped_gvs = [(tf.clip_by_value(grad, -args.grad_clip, args.grad_clip), var) for grad, var in gvs]
        optimizer = optimizer.apply_gradients(capped_gvs)

        embeddings_norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
        normalized_embeddings = embeddings / embeddings_norm
    processing_message_lst = list()
    with tf.Session(graph=graph) as sess:

        tf.global_variables_initializer().run()
        for e in range(args.num_epochs):
            data_loader.reset_batch_pointer()
            for b in range(data_loader.num_batches):
                start = time.time()
                x, y = data_loader.next_batch()
                feed = {input_data: x, targets: y}
                train_loss, _ = sess.run([loss, optimizer], feed)
                end = time.time()

                processing_message = "{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
                    b, data_loader.num_batches,
                    e, train_loss, end - start)

                print(processing_message)
                processing_message_lst.append(processing_message)
                # print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}".format(
                #     b, data_loader.num_batches,
				#     e, train_loss, end - start))


            np.save('nnlm_word_embeddings.zh', normalized_embeddings.eval())

    # record training processing
    print(start - end)
    local_time = str(time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()))
    with open("{}.txt".format('casdsa'), 'w', encoding='utf-8') as f:
        f.write(local_time)
        f.write(args_msg)
        f.write('\n'.join(processing_message_lst))
Ejemplo n.º 51
0
  def body(self, features):
    hparams = self.hparams
    batch_size = common_layers.shape_list(features["inputs"])[0]

    # Swap time and batch axes.
    input_frames = common_video.swap_time_and_batch_axes(features["inputs"])
    target_frames = common_video.swap_time_and_batch_axes(features["targets"])

    # Get actions if exist otherwise use zeros
    input_actions = self.get_input_if_exists(
        features, "input_action", batch_size, hparams.video_num_input_frames)
    target_actions = self.get_input_if_exists(
        features, "target_action", batch_size, hparams.video_num_target_frames)

    # Get rewards if exist otherwise use zeros
    input_rewards = self.get_input_if_exists(
        features, "input_reward", batch_size, hparams.video_num_input_frames)
    target_rewards = self.get_input_if_exists(
        features, "target_reward", batch_size, hparams.video_num_target_frames)

    all_actions = tf.concat([input_actions, target_actions], axis=0)
    all_rewards = tf.concat([input_rewards, target_rewards], axis=0)
    all_frames = tf.concat([input_frames, target_frames], axis=0)

    # Each image is being used twice, in latent tower and main tower.
    # This is to make sure we are using the *same* image for both, ...
    # ... given how TF queues work.
    # NOT sure if this is required at all. Doesn"t hurt though! :)
    all_frames = tf.identity(all_frames)

    gen_images, gen_rewards, latent_means, latent_stds = self.construct_model(
        images=all_frames,
        actions=all_actions,
        rewards=all_rewards,
    )

    extra_loss = self.get_extra_loss(
        latent_means=latent_means,
        latent_stds=latent_stds,
        true_frames=all_frames,
        gen_frames=gen_images)

    # Visualize predictions in Tensorboard
    if self.is_training:
      self.visualize_predictions(all_frames[1:], gen_images)

    # Ignore the predictions from the input frames.
    # This is NOT the same as original paper/implementation.
    predictions = gen_images[hparams.video_num_input_frames-1:]
    reward_pred = gen_rewards[hparams.video_num_input_frames-1:]
    reward_pred = tf.squeeze(reward_pred, axis=2)  # Remove extra dimension.

    # Swap back time and batch axes.
    predictions = common_video.swap_time_and_batch_axes(predictions)
    reward_pred = common_video.swap_time_and_batch_axes(reward_pred)

    if self.is_training and hparams.internal_loss:
      # add the loss for input frames as well.
      extra_gts = all_frames[1:hparams.video_num_input_frames]
      extra_gts = common_video.swap_time_and_batch_axes(extra_gts)
      extra_pds = gen_images[:hparams.video_num_input_frames-1]
      extra_pds = common_video.swap_time_and_batch_axes(extra_pds)
      extra_raw_gts = features["inputs_raw"][:, 1:]
      recon_loss = self.get_extra_internal_loss(
          extra_raw_gts, extra_gts, extra_pds)
      extra_loss += recon_loss

    return_targets = predictions
    if hparams.reward_prediction:
      return_targets = {"targets": predictions, "target_reward": reward_pred}

    return return_targets, extra_loss
Ejemplo n.º 52
0
def get_model(point_cloud, is_training, bn_decay=None):
    """ Classification PointNet, input is BxNx3, output BxNx50 """
    batch_size = point_cloud.get_shape()[0].value  # batch_size
    num_point = point_cloud.get_shape()[1].value  # num_point
    end_points = {}

    with tf.variable_scope('transform_net1') as sc:
        transform = input_transform_net(point_cloud,
                                        is_training,
                                        bn_decay,
                                        K=3)
    point_cloud_transformed = tf.matmul(point_cloud, transform)
    input_image = tf.expand_dims(point_cloud_transformed, -1)

    net = tf_util.conv2d(input_image,
                         64, [1, 3],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv1',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv2',
                         bn_decay=bn_decay)

    with tf.variable_scope('transform_net2') as sc:
        transform = feature_transform_net(net, is_training, bn_decay, K=64)
    end_points['transform'] = transform
    net_transformed = tf.matmul(tf.squeeze(net, axis=[2]), transform)
    point_feat = tf.expand_dims(net_transformed, [2])
    print(point_feat)

    net = tf_util.conv2d(point_feat,
                         64, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv3',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv4',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         1024, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv5',
                         bn_decay=bn_decay)
    global_feat = tf_util.max_pool2d(net, [num_point, 1],
                                     padding='VALID',
                                     scope='maxpool')
    print(global_feat)

    global_feat_expand = tf.tile(global_feat, [1, num_point, 1, 1])
    concat_feat = tf.concat(3, [point_feat, global_feat_expand])
    print(concat_feat)
    # 定义分割的mpl512-256-128  128-m, m为点所属的类别数目
    net = tf_util.conv2d(concat_feat,
                         512, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv6',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         256, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv7',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv8',
                         bn_decay=bn_decay)
    net = tf_util.conv2d(net,
                         128, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         bn=True,
                         is_training=is_training,
                         scope='conv9',
                         bn_decay=bn_decay)

    net = tf_util.conv2d(net,
                         50, [1, 1],
                         padding='VALID',
                         stride=[1, 1],
                         activation_fn=None,
                         scope='conv10')
    net = tf.squeeze(net, [2])  # BxNxC

    return net, end_points
Ejemplo n.º 53
0
    def test_model_serving(self):
        """
        Train a simple model and test serving flow by loading the SavedModel
        """

        # Test model training on TFRecord SequenceExample data
        data_dir = os.path.join(self.root_data_dir, "tfrecord")
        feature_config: FeatureConfig = self.get_feature_config()

        metrics_keys = ["categorical_accuracy"]

        def get_dataset(parse_tfrecord):
            return RelevanceDataset(
                data_dir=data_dir,
                data_format=DataFormatKey.TFRECORD,
                feature_config=feature_config,
                tfrecord_type=self.args.tfrecord_type,
                max_sequence_size=self.args.max_sequence_size,
                batch_size=self.args.batch_size,
                preprocessing_keys_to_fns={},
                train_pcent_split=self.args.train_pcent_split,
                val_pcent_split=self.args.val_pcent_split,
                test_pcent_split=self.args.test_pcent_split,
                use_part_files=self.args.use_part_files,
                parse_tfrecord=parse_tfrecord,
                file_io=self.file_io,
                logger=self.logger,
            )

        # Get raw TFRecord dataset
        raw_dataset = get_dataset(parse_tfrecord=False)

        # Parse the raw TFRecord dataset
        parsed_dataset = get_dataset(parse_tfrecord=True)

        model: RankingModel = self.get_ranking_model(
            loss_key=self.args.loss_key, feature_config=feature_config, metrics_keys=metrics_keys
        )

        model.fit(dataset=parsed_dataset, num_epochs=1, models_dir=self.output_dir)

        model.save(
            models_dir=self.args.models_dir,
            preprocessing_keys_to_fns={},
            postprocessing_fn=None,
            required_fields_only=not self.args.use_all_fields_at_inference,
            pad_sequence=self.args.pad_sequence_at_inference,
        )

        # Load SavedModel and get the right serving signature
        default_model = kmodels.load_model(
            os.path.join(self.output_dir, "final", "default"), compile=False
        )
        assert ServingSignatureKey.DEFAULT in default_model.signatures
        default_signature = default_model.signatures[ServingSignatureKey.DEFAULT]

        tfrecord_model = kmodels.load_model(
            os.path.join(self.output_dir, "final", "tfrecord"), compile=False
        )
        assert ServingSignatureKey.TFRECORD in tfrecord_model.signatures
        tfrecord_signature = tfrecord_model.signatures[ServingSignatureKey.TFRECORD]

        # Fetch a single batch for testing
        sequence_example_protos = next(iter(raw_dataset.test))
        parsed_sequence_examples = next(iter(parsed_dataset.test))[0]
        parsed_dataset_batch = parsed_dataset.test.take(1)

        # Use the loaded serving signatures for inference
        model_predictions = model.predict(parsed_dataset_batch)[self.args.output_name].values
        default_signature_predictions = default_signature(**parsed_sequence_examples)[
            self.args.output_name
        ]

        # Since we do not pad dummy records in tfrecord serving signature,
        # we can only predict on a single record at a time
        tfrecord_signature_predictions = [
            tfrecord_signature(protos=tf.gather(sequence_example_protos, [i]))[
                self.args.output_name
            ]
            for i in range(self.args.batch_size)
        ]

        # Get mask for padded values
        mask = flatten_query(parsed_sequence_examples["mask"])

        # Flatten scores to each record and filter out scores from padded records
        default_signature_predictions = tf.squeeze(
            filter_records(flatten_query(default_signature_predictions), mask), axis=-1
        )
        tfrecord_signature_predictions = tf.squeeze(
            tf.concat(tfrecord_signature_predictions, axis=1)
        )

        # Compare the scores from the different versions of the model
        assert np.isclose(model_predictions, default_signature_predictions, rtol=0.01,).all()

        assert np.isclose(model_predictions, tfrecord_signature_predictions, rtol=0.01,).all()

        assert np.isclose(
            default_signature_predictions, tfrecord_signature_predictions, rtol=0.01,
        ).all()
Ejemplo n.º 54
0
def VINet_final(opt,
                masked_img,
                mask,
                prev_state=None,
                prev_feed=None,
                idx=0,
                trainable=True,
                namescope='module'):
    with tf.variable_scope(namescope):
        # masked_img: b x C x TxHxW
        T = masked_img.get_shape().as_list()[2]
        ref_idx = (T - 1) // 2
        ones = tf.ones_like(mask)
        # encoder
        enc_output = []
        enc_input = tf.concat([masked_img, ones, ones * mask], axis=1)
        # print(type(enc_input[:,:,ref_idx,:,:]), type(mask))
        # input('hereeeee')

        f1, f1_64, f1_128, f1_256 = VI_2D_Encoder_3(enc_input[:, :,
                                                              ref_idx, :, :],
                                                    opt,
                                                    reuse=False,
                                                    trainable=trainable,
                                                    namescope='encoder1')

        f2, f2_64, f2_128, _ = VI_2D_Encoder_3(enc_input[:, :,
                                                         ref_idx - 2, :, :],
                                               opt,
                                               reuse=False,
                                               trainable=trainable,
                                               namescope='encoder2')
        f3, f3_64, f3_128, _ = VI_2D_Encoder_3(enc_input[:, :,
                                                         ref_idx - 1, :, :],
                                               opt,
                                               reuse=True,
                                               trainable=trainable,
                                               namescope='encoder2')
        f4, f4_64, f4_128, _ = VI_2D_Encoder_3(enc_input[:, :,
                                                         ref_idx + 1, :, :],
                                               opt,
                                               reuse=True,
                                               trainable=trainable,
                                               namescope='encoder2')
        f5, f5_64, f5_128, _ = VI_2D_Encoder_3(enc_input[:, :,
                                                         ref_idx + 2, :, :],
                                               opt,
                                               reuse=True,
                                               trainable=trainable,
                                               namescope='encoder2')
        f6, f6_64, f6_128, f6_256 = VI_2D_Encoder_3(prev_feed,
                                                    opt,
                                                    reuse=True,
                                                    trainable=trainable,
                                                    namescope='encoder2')

        flow2 = LongFlowNetCorr(f1,
                                f2,
                                opt,
                                trainable=trainable,
                                reuse=False,
                                namescope='flownet')
        flow3 = LongFlowNetCorr(f1,
                                f3,
                                opt,
                                trainable=trainable,
                                reuse=True,
                                namescope='flownet')
        flow4 = LongFlowNetCorr(f1,
                                f4,
                                opt,
                                trainable=trainable,
                                reuse=True,
                                namescope='flownet')
        flow5 = LongFlowNetCorr(f1,
                                f5,
                                opt,
                                trainable=trainable,
                                reuse=True,
                                namescope='flownet')
        flow6 = LongFlowNetCorr(f1,
                                f6,
                                opt,
                                trainable=trainable,
                                reuse=True,
                                namescope='flownet')

        f2_warp = WarpingLayer(f2, flow2)
        f3_warp = WarpingLayer(f3, flow3)
        f4_warp = WarpingLayer(f4, flow4)
        f5_warp = WarpingLayer(f5, flow5)
        f6_warp = WarpingLayer(f6, flow6)

        f_stack_oth = tf.stack([f2_warp, f3_warp, f4_warp, f5_warp, f6_warp],
                               axis=2)
        f_agg = tf.squeeze(VI_Aggregator(f_stack_oth,
                                         depth=5,
                                         trainable=trainable,
                                         namescope='st_agg'),
                           axis=2)
        occlusion_mask = MaskEstimator_(tf.abs(f1 - f_agg),
                                        trainable=trainable,
                                        namescope='masknet')
        f_syn = (1 - occlusion_mask) * f1 + occlusion_mask * f_agg

        bott_input = tf.concat([f1, f_syn], axis=1)
        output = VI_2D_BottleNeck(bott_input,
                                  trainable=trainable,
                                  namescope='bottleneck')

        # CONV LSTM
        state = ConvLSTM(output,
                         prev_state,
                         trainable=trainable,
                         namescope='convlstm')

        bot_h, bot_w = output.get_shape().as_list()[2:4]
        # ============================ SCALE - 1/4 : 64 =============================
        # align_corners=True
        flow2_64 = resize_bilinear(flow2, (bot_h * 2, bot_w * 2),
                                   align_corners=True) * 2
        flow3_64 = resize_bilinear(flow3, (bot_h * 2, bot_w * 2),
                                   align_corners=True) * 2
        flow4_64 = resize_bilinear(flow4, (bot_h * 2, bot_w * 2),
                                   align_corners=True) * 2
        flow5_64 = resize_bilinear(flow5, (bot_h * 2, bot_w * 2),
                                   align_corners=True) * 2
        flow6_64 = resize_bilinear(flow6, (bot_h * 2, bot_w * 2),
                                   align_corners=True) * 2

        f2_64_warp = WarpingLayer(f2_64, flow2_64)
        f3_64_warp = WarpingLayer(f3_64, flow3_64)
        f4_64_warp = WarpingLayer(f4_64, flow4_64)
        f5_64_warp = WarpingLayer(f5_64, flow5_64)
        f6_64_warp = WarpingLayer(f6_64, flow6_64)

        flow2_64 = LongFlowNetCorr(f1_64,
                                   f2_64_warp,
                                   opt,
                                   flow2_64,
                                   trainable=trainable,
                                   reuse=False,
                                   namescope='flownet_64') + flow2_64
        flow3_64 = LongFlowNetCorr(f1_64,
                                   f3_64_warp,
                                   opt,
                                   flow3_64,
                                   trainable=trainable,
                                   reuse=True,
                                   namescope='flownet_64') + flow3_64
        flow4_64 = LongFlowNetCorr(f1_64,
                                   f4_64_warp,
                                   opt,
                                   flow4_64,
                                   trainable=trainable,
                                   reuse=True,
                                   namescope='flownet_64') + flow4_64
        flow5_64 = LongFlowNetCorr(f1_64,
                                   f5_64_warp,
                                   opt,
                                   flow5_64,
                                   trainable=trainable,
                                   reuse=True,
                                   namescope='flownet_64') + flow5_64
        flow6_64 = LongFlowNetCorr(f1_64,
                                   f6_64_warp,
                                   opt,
                                   flow6_64,
                                   trainable=trainable,
                                   reuse=True,
                                   namescope='flownet_64') + flow6_64

        f2_64_warp = WarpingLayer(f2_64, flow2_64)
        f3_64_warp = WarpingLayer(f3_64, flow3_64)
        f4_64_warp = WarpingLayer(f4_64, flow4_64)
        f5_64_warp = WarpingLayer(f5_64, flow5_64)
        f6_64_warp = WarpingLayer(f6_64, flow6_64)

        f_stack_64_oth = tf.stack(
            [f2_64_warp, f3_64_warp, f4_64_warp, f5_64_warp, f6_64_warp],
            axis=2)
        f_agg_64 = tf.squeeze(VI_Aggregator(f_stack_64_oth,
                                            depth=5,
                                            trainable=trainable,
                                            namescope='st_agg_64'),
                              axis=2)
        occlusion_mask_64 = MaskEstimator_(tf.abs(f1_64 - f_agg_64),
                                           trainable=trainable,
                                           namescope='masknet_64')
        f_syn_64 = (1 -
                    occlusion_mask_64) * f1_64 + occlusion_mask_64 * f_agg_64

        # ============================= SCALE - 1/2 : 128 ===============================
        flow2_128 = resize_bilinear(flow2_64, (bot_h * 4, bot_w * 4),
                                    align_corners=True) * 2
        flow3_128 = resize_bilinear(flow3_64, (bot_h * 4, bot_w * 4),
                                    align_corners=True) * 2
        flow4_128 = resize_bilinear(flow4_64, (bot_h * 4, bot_w * 4),
                                    align_corners=True) * 2
        flow5_128 = resize_bilinear(flow5_64, (bot_h * 4, bot_w * 4),
                                    align_corners=True) * 2
        flow6_128 = resize_bilinear(flow6_64, (bot_h * 4, bot_w * 4),
                                    align_corners=True) * 2

        f2_128_warp = WarpingLayer(f2_128, flow2_128)
        f3_128_warp = WarpingLayer(f3_128, flow3_128)
        f4_128_warp = WarpingLayer(f4_128, flow4_128)
        f5_128_warp = WarpingLayer(f5_128, flow5_128)
        f6_128_warp = WarpingLayer(f6_128, flow6_128)

        flow2_128 = LongFlowNetCorr(f1_128,
                                    f2_128_warp,
                                    opt,
                                    flow2_128,
                                    trainable=trainable,
                                    reuse=False,
                                    namescope='flownet_128') + flow2_128
        flow3_128 = LongFlowNetCorr(f1_128,
                                    f3_128_warp,
                                    opt,
                                    flow3_128,
                                    trainable=trainable,
                                    reuse=True,
                                    namescope='flownet_128') + flow3_128
        flow4_128 = LongFlowNetCorr(f1_128,
                                    f4_128_warp,
                                    opt,
                                    flow4_128,
                                    trainable=trainable,
                                    reuse=True,
                                    namescope='flownet_128') + flow4_128
        flow5_128 = LongFlowNetCorr(f1_128,
                                    f5_128_warp,
                                    opt,
                                    flow5_128,
                                    trainable=trainable,
                                    reuse=True,
                                    namescope='flownet_128') + flow5_128
        flow6_128 = LongFlowNetCorr(f1_128,
                                    f6_128_warp,
                                    opt,
                                    flow6_128,
                                    trainable=trainable,
                                    reuse=True,
                                    namescope='flownet_128') + flow6_128

        f2_128_warp = WarpingLayer(f2_128, flow2_128)
        f3_128_warp = WarpingLayer(f3_128, flow3_128)
        f4_128_warp = WarpingLayer(f4_128, flow4_128)
        f5_128_warp = WarpingLayer(f5_128, flow5_128)
        f6_128_warp = WarpingLayer(f6_128, flow6_128)

        f_stack_128_oth = tf.stack(
            [f2_128_warp, f3_128_warp, f4_128_warp, f5_128_warp, f6_128_warp],
            axis=2)
        f_agg_128 = tf.squeeze(VI_Aggregator(f_stack_128_oth,
                                             depth=5,
                                             trainable=trainable,
                                             namescope='st_agg_128'),
                               axis=2)
        occlusion_mask_128 = MaskEstimator_(tf.abs(f1_128 - f_agg_128),
                                            trainable=trainable,
                                            namescope='masknet_128')
        f_syn_128 = (
            1 - occlusion_mask_128) * f1_128 + occlusion_mask_128 * f_agg_128

        output, _ = VI_2D_Decoder_3(tf.expand_dims(state[0], axis=2),
                                    opt,
                                    x2_64_warp=tf.expand_dims(f_syn_64,
                                                              axis=2),
                                    x2_128_warp=tf.expand_dims(f_syn_128,
                                                               axis=2),
                                    trainable=trainable,
                                    namescope='decoder')
        occ_mask = resize_bilinear(occlusion_mask, (bot_h * 8, bot_w * 8),
                                   align_corners=True)
        occ_mask_64 = resize_bilinear(occlusion_mask_64,
                                      (bot_h * 4, bot_w * 4),
                                      align_corners=True)
        occ_mask_128 = resize_bilinear(occlusion_mask_128,
                                       (bot_h * 2, bot_w * 2),
                                       align_corners=True)

        flow6_256, flow6_512 = None, None
        # NOTE: TF is static graph.
        if opt.prev_warp:
            if prev_state is not None or idx != 0:
                h, w = flow6_128.get_shape().as_list()[2:4]
                flow6_256 = resize_bilinear(flow6_128, (h * 2, w * 2)) * 2
                flow6_512 = resize_bilinear(flow6_128, (h * 4, w * 4)) * 4
                f6_256_warp = WarpingLayer(f6_256, flow6_256)
                flow6_256 = LongFlowNetCorr(f1_256,
                                            f6_256_warp,
                                            opt,
                                            flow6_256,
                                            trainable=trainable,
                                            namescope='flownet_256')
                occlusion_mask_256 = MaskEstimator_(tf.abs(f1_256 -
                                                           f6_256_warp),
                                                    trainable=trainable,
                                                    namescope='masknet_256')
                output_ = output
                if opt.double_size:
                    prev_feed_warp = WarpingLayer(prev_feed[:, :3], flow6_512)
                    h, w = occlusion_mask_256.get_shape().as_list()[2:4]
                    occlusion_mask_512 = resize_nearest(
                        occlusion_mask_256, (h * 2, w * 2))
                    output = tf.squeeze(
                        (1 - tf.expand_dims(occlusion_mask_512, axis=2)) *
                        output + occlusion_mask_512 *
                        tf.expand_dims(prev_feed_warp, axis=2),
                        axis=2)
                    flow6_256 = flow6_512
                else:
                    prev_feed_warp = WarpingLayer(prev_feed[:, :3], flow6_256)
                    output = tf.squeeze(
                        (1 - tf.expand_dims(occlusion_mask_256, axis=2)) *
                        output + occlusion_mask_256 *
                        tf.expand_dims(prev_feed_warp, axis=2),
                        axis=2)
                if opt.loss_on_raw:
                    output = (output, output_)
        # return output, tf.stack([flow2_128,flow3_128,flow4_128,flow5_128, flow6_128],axis=2), state, tf.stack([occ_mask, 1-occ_mask, occ_mask_64, 1-occ_mask_64, occ_mask_128, 1-occ_mask_128], axis=2), flow6_256
        return output, state[0], state[1]
Ejemplo n.º 55
0
    def call(self, inputs, cps=None, isFirstLayer=False):

        if isFirstLayer:
            tf.print("computed cp for the last input", cps[-1])
            bOnA = inputs - self.firstLayerTA0 - cps / (self.firstLayerK1M *
                                                        self.E0)
            tf.print("computed bOna for the last input", bOnA[-1])
            tf.print("second parts",
                     (4 * inputs * cps / (self.firstLayerK1M * self.E0))[-1])
            tf.assert_rank(
                self.firstLayerK1M * self.E0 * self.firstLayerTA0 / cps,
                tf.rank(inputs))
            olderInput = tf.where(
                tf.equal(
                    self.firstLayerK1M * self.E0 * self.firstLayerTA0 / cps,
                    0), inputs,
                0.5 * (bOnA + (tf.pow(bOnA, 2) + 4 * inputs * cps /
                               (self.firstLayerK1M * self.E0))**0.5))
            tf.print("solution for the last inputs of first layer: ",
                     olderInput[-1])
            olderInput = self.firstLayerk2 * self.firstLayerK1M / self.firstLayerkdT * self.firstLayerTA0 * self.E0 / cps * olderInput / (
                1 + self.firstLayerK1M * self.E0 / cps * olderInput)
        else:
            olderInput = inputs

        #For batch computation: k1m * tf.transpose(inputs) doesn't work and we need to add a 1 axis in the end and use only *
        #Just above we use rank1 vector so should keep rank2 batches of input
        cpsExpand = tf.expand_dims(cps, -1)
        tf.assert_rank(cpsExpand, 3)
        tf.assert_rank(cps, 2)
        olderInputExpand = tf.expand_dims(olderInput, -1)
        tf.assert_rank(olderInputExpand, 3)
        olderInputMidExpand = tf.expand_dims(olderInput, 1)

        Cactivs = tf.where(
            self.mask > 0, self.Cactiv /
            (1 + self.k1M * self.E0 * olderInputExpand / cpsExpand), 0)
        Cinhibs = tf.where(
            self.mask < 0, self.Cinhib /
            (1 + self.k3M * self.E0 * olderInputExpand / cpsExpand), 0)
        tf.assert_rank(Cactivs, 3)
        tf.assert_rank(Cinhibs, 3)
        Inhib = tf.squeeze(tf.matmul(olderInputMidExpand, Cinhibs),
                           axis=1) / self.kdT
        x_eq_clipped = tf.squeeze(tf.matmul(olderInputMidExpand, Cactivs),
                                  axis=1) / (self.kdI * cps + Inhib / cps)

        #unclipped version:
        Cactivs_unclipped = tf.where(
            self.kernel > 0, self.Cactiv * self.kernel /
            (1 + self.k1M * self.E0 * olderInputExpand / cpsExpand), 0)
        Cinhibs_unclipped = tf.where(
            self.kernel < 0, (-1) * self.Cinhib * self.kernel /
            (1 + self.k3M * self.E0 * olderInputExpand / cpsExpand), 0)
        tf.assert_rank(Cactivs_unclipped, 3)
        tf.assert_rank(Cinhibs_unclipped, 3)
        #CAREFUL: now the cactivs has taken the batch size, it is of rank 3 : [None,inputdims,outputdims]
        # THUS WE NEED: [None,1,inputdims] to use the matmul, and then squeeze the result!
        Inhib_unclipped = tf.squeeze(tf.matmul(olderInputMidExpand,
                                               Cinhibs_unclipped),
                                     axis=1) / self.kdT
        x_eq_unclipped = tf.squeeze(
            tf.matmul(olderInputMidExpand, Cactivs_unclipped),
            axis=1) / (self.kdI * cps + Inhib_unclipped / cps)
        tf.assert_rank(tf.squeeze(tf.matmul(olderInputMidExpand,
                                            Cinhibs_unclipped),
                                  axis=1),
                       2,
                       message="compute not good dims")

        tf.print("solution from an unknown layer:", x_eq_clipped[-1])

        outputs = tf.stop_gradient(x_eq_clipped -
                                   x_eq_unclipped) + x_eq_unclipped
        tf.assert_rank(outputs, 2, message="outputs not good dims")
        return outputs
def inception_v1(inputs,
                 num_classes=1000,
                 is_training=True,
                 dropout_keep_prob=0.8,
                 prediction_fn=slim.softmax,
                 spatial_squeeze=True,
                 reuse=None,
                 scope='InceptionV1',
                 global_pool=False):
  """Defines the Inception V1 architecture.

  This architecture is defined in:

    Going deeper with convolutions
    Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed,
    Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich.
    http://arxiv.org/pdf/1409.4842v1.pdf.

  The default image size used to train this network is 224x224.

  Args:
    inputs: a tensor of size [batch_size, height, width, channels].
    num_classes: number of predicted classes. If 0 or None, the logits layer
      is omitted and the input features to the logits layer (before dropout)
      are returned instead.
    is_training: whether is training or not.
    dropout_keep_prob: the percentage of activation values that are retained.
    prediction_fn: a function to get predictions out of logits.
    spatial_squeeze: if True, logits is of shape [B, C], if false logits is of
        shape [B, 1, 1, C], where B is batch_size and C is number of classes.
    reuse: whether or not the network and its variables should be reused. To be
      able to reuse 'scope' must be given.
    scope: Optional variable_scope.
    global_pool: Optional boolean flag to control the avgpooling before the
      logits layer. If false or unset, pooling is done with a fixed window
      that reduces default-sized inputs to 1x1, while larger inputs lead to
      larger outputs. If true, any input size is pooled down to 1x1.

  Returns:
    net: a Tensor with the logits (pre-softmax activations) if num_classes
      is a non-zero integer, or the non-dropped-out input to the logits layer
      if num_classes is 0 or None.
    end_points: a dictionary from components of the network to the corresponding
      activation.
  """
  # Final pooling and prediction
  with tf.variable_scope(scope, 'InceptionV1', [inputs], reuse=reuse) as scope:
    with slim.arg_scope([slim.batch_norm, slim.dropout],
                        is_training=is_training):
      net, end_points = inception_v1_base(inputs, scope=scope)
      with tf.variable_scope('Logits'):
        if global_pool:
          # Global average pooling.
          net = tf.reduce_mean(net, [1, 2], keep_dims=True, name='global_pool')
          end_points['global_pool'] = net
        else:
          # Pooling with a fixed kernel size.
          net = slim.avg_pool2d(net, [7, 7], stride=1, scope='AvgPool_0a_7x7')
          end_points['AvgPool_0a_7x7'] = net
        if not num_classes:
          return net, end_points
        net = slim.dropout(net, dropout_keep_prob, scope='Dropout_0b')
        logits = slim.conv2d(net, num_classes, [1, 1], activation_fn=None,
                             normalizer_fn=None, scope='Conv2d_0c_1x1')
        if spatial_squeeze:
          logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze')

        end_points['Logits'] = logits
        end_points['Predictions'] = prediction_fn(logits, scope='Predictions')
  return logits, end_points
Ejemplo n.º 57
0
    def inference(self):
        self.input_x1_embed = tf.nn.embedding_lookup(self.Embedding,
                                                     self.input_x1)
        self.input_x2_embed = tf.nn.embedding_lookup(self.Embedding,
                                                     self.input_x2)

        with tf.variable_scope("lstm") as scope:
            lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_size)
            lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(self.hidden_size)
            # with tf.variable_scope("rnn1"):
            outputs1, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,
                                                          lstm_bw_cell,
                                                          self.input_x1_embed,
                                                          dtype=tf.float32)
            outputs1_rnn = tf.concat(outputs1, axis=2)

            self.outputs1_last = outputs1_rnn[:, -1, :]
            scope.reuse_variables()
            outputs2, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,
                                                          lstm_bw_cell,
                                                          self.input_x2_embed,
                                                          dtype=tf.float32)

            outputs2_rnn = tf.concat(outputs2, axis=2)
            self.outputs2_last = outputs2_rnn[:, -1, :]

        with tf.variable_scope("outputs"):
            # self.conc = tf.concat([self.outputs1_last, self.outputs2_last], axis=1)
            self.manha = tf.reduce_sum(tf.abs(
                tf.subtract(self.outputs1_last, self.outputs2_last)),
                                       axis=1,
                                       keepdims=True)
            self.squre = tf.reduce_sum(tf.square(
                tf.subtract(self.outputs1_last, self.outputs2_last)),
                                       axis=1,
                                       keepdims=True)

            self.element_wise = tf.multiply(self.outputs1_last,
                                            self.outputs2_last)
            self.norm1 = tf.sqrt(
                tf.reduce_sum(tf.square(self.outputs1_last),
                              axis=1,
                              keepdims=True))
            self.norm2 = tf.sqrt(
                tf.reduce_sum(tf.square(self.outputs2_last),
                              axis=1,
                              keepdims=True))
            self.sum12 = tf.reduce_sum(self.element_wise,
                                       axis=1,
                                       keepdims=True)
            self.cos = tf.divide(self.sum12,
                                 tf.multiply(self.norm1, self.norm2))

            self.input_dense = tf.concat([
                self.element_wise, self.manha, self.norm1, self.norm2,
                self.sum12, self.cos, self.squre
            ],
                                         axis=1)

            self.fc1 = tf.layers.dense(self.input_dense,
                                       256,
                                       activation=tf.nn.relu)
            self.fc1 = tf.nn.dropout(self.fc1,
                                     keep_prob=self.dropout_keep_prob)

            self.fc2 = tf.layers.dense(self.fc1, 128, activation=tf.nn.relu)
            self.fc2 = tf.nn.dropout(self.fc2,
                                     keep_prob=self.dropout_keep_prob)

            self.fc3 = tf.layers.dense(self.fc2, 32, activation=tf.nn.relu)
            self.fc3 = tf.nn.dropout(self.fc3,
                                     keep_prob=self.dropout_keep_prob)

            self.logits = tf.squeeze(tf.layers.dense(self.fc3,
                                                     1,
                                                     activation=tf.nn.sigmoid),
                                     axis=1,
                                     name="predict")
        return self.logits
Ejemplo n.º 58
0
Archivo: FCN.py Proyecto: sysudhh/fcn
def main(argv=None):
    keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
    image = tf.placeholder(tf.float32,
                           shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
                           name="input_image")
    annotation = tf.placeholder(tf.int32,
                                shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
                                name="annotation")

    pred_annotation, logits = inference(image, keep_probability)
    tf.summary.image("input_image", image, max_outputs=2)
    tf.summary.image("ground_truth",
                     tf.cast(annotation, tf.uint8),
                     max_outputs=2)
    tf.summary.image("pred_annotation",
                     tf.cast(pred_annotation, tf.uint8),
                     max_outputs=2)
    loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(
        logits=logits,
        labels=tf.squeeze(annotation, squeeze_dims=[3]),
        name="entropy")))
    tf.summary.scalar("entropy", loss)

    trainable_var = tf.trainable_variables()
    if FLAGS.debug:
        for var in trainable_var:
            utils.add_to_regularization_and_summary(var)
    train_op = train(loss, trainable_var)

    print("Setting up summary op...")
    summary_op = tf.summary.merge_all()

    print("Setting up image reader...")
    train_records = scene_parsing.read_dataset(FLAGS.data_dir)
    print(len(train_records))
    #print(len(valid_records))

    print("Setting up dataset reader")
    image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
    if FLAGS.mode == 'train':
        train_dataset_reader = dataset.BatchDatset(train_records,
                                                   image_options)
    #validation_dataset_reader = dataset.BatchDatset(valid_records, image_options)

    sess = tf.Session()

    print("Setting up Saver...")
    saver = tf.train.Saver()
    summary_writer = tf.summary.FileWriter(FLAGS.logs_dir, sess.graph)

    sess.run(tf.global_variables_initializer())
    ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)
        print("Model restored...")

    if FLAGS.mode == "train":
        for itr in xrange(MAX_ITERATION):
            train_images, train_annotations = train_dataset_reader.next_batch(
                FLAGS.batch_size)
            feed_dict = {
                image: train_images,
                annotation: train_annotations,
                keep_probability: 0.85
            }

            sess.run(train_op, feed_dict=feed_dict)

            if itr % 10 == 0:
                train_loss, summary_str = sess.run([loss, summary_op],
                                                   feed_dict=feed_dict)
                print("Step: %d, Train_loss:%g" % (itr, train_loss))
                summary_writer.add_summary(summary_str, itr)

            if itr % 500 == 0:
                # valid_images, valid_annotations = validation_dataset_reader.next_batch(FLAGS.batch_size)
                # valid_loss = sess.run(loss, feed_dict={image: valid_images, annotation: valid_annotations,
                #                                        keep_probability: 1.0})
                # print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss))
                saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr)

    elif FLAGS.mode == "visualize":
        valid_images, valid_annotations = train_dataset_reader.get_random_batch(
            FLAGS.batch_size)
        pred = sess.run(pred_annotation,
                        feed_dict={
                            image: valid_images,
                            annotation: valid_annotations,
                            keep_probability: 1.0
                        })
        valid_annotations = np.squeeze(valid_annotations, axis=3)
        pred = np.squeeze(pred, axis=3)

        for itr in range(FLAGS.batch_size):
            utils.save_image(valid_images[itr].astype(np.uint8),
                             FLAGS.logs_dir,
                             name="inp_" + str(5 + itr))
            utils.save_image(valid_annotations[itr].astype(np.uint8),
                             FLAGS.logs_dir,
                             name="gt_" + str(5 + itr))
            utils.save_image(pred[itr].astype(np.uint8),
                             FLAGS.logs_dir,
                             name="pred_" + str(5 + itr))
            print("Saved image: %d" % itr)
Ejemplo n.º 59
0
env = ArmEnv(size_x=4,
             size_y=3,
             cubes_cnt=4,
             episode_max_length=2000,
             finish_reward=200,
             action_minus_reward=0.0,
             tower_target_size=3)  #is 0.1 IMPORTANT???????????

s = env.reset()
obs_len = len(s)

tf.reset_default_graph()

state = tf.placeholder('float32', shape=[None, obs_len], name="STATE")
actions = tf.squeeze(tf.placeholder('int32', name="ACTIONS"))
'''
============================
Actor = Policy Approxiamtion
============================
'''
q_estimation = tf.placeholder('float32', name="Q-Estimation")

inp = tf.layers.dense(state,
                      10,
                      name="INPUT",
                      kernel_initializer=tf.random_normal_initializer(
                          mean=0, stddev=0.1),
                      bias_initializer=tf.initializers.constant(0))

out = tf.layers.dense(inp,
Ejemplo n.º 60
0
    def build_model(self):
        # Kernel initialization for the convolutions
        self.init_kernel = tf.random_normal_initializer(mean=0.0, stddev=0.02)
        # Placeholders
        self.is_training = tf.placeholder(tf.bool)
        self.image_input = tf.placeholder(tf.float32,
                                          shape=[None] +
                                          self.config.trainer.image_dims,
                                          name="x")
        self.noise_tensor = tf.placeholder(
            tf.float32,
            shape=[None, self.config.trainer.noise_dim],
            name="noise")
        self.true_labels = tf.placeholder(dtype=tf.float32,
                                          shape=[None, 1],
                                          name="true_labels")
        self.generated_labels = tf.placeholder(dtype=tf.float32,
                                               shape=[None, 1],
                                               name="gen_labels")
        self.real_noise = tf.placeholder(dtype=tf.float32,
                                         shape=[None] +
                                         self.config.trainer.image_dims,
                                         name="real_noise")
        self.fake_noise = tf.placeholder(dtype=tf.float32,
                                         shape=[None] +
                                         self.config.trainer.image_dims,
                                         name="fake_noise")

        self.logger.info("Building training graph...")
        with tf.variable_scope("BIGAN"):
            with tf.variable_scope("Encoder_Model"):
                self.noise_gen = self.encoder(self.image_input)

            with tf.variable_scope("Generator_Model"):
                self.image_gen = self.generator(
                    self.noise_tensor) + self.fake_noise
                self.reconstructed = self.generator(self.noise_gen)

            with tf.variable_scope("Discriminator_Model"):
                # E(x) and x --> This being real is the output of discriminator
                l_encoder, inter_layer_inp = self.discriminator(
                    self.noise_gen, self.image_input + self.real_noise)
                # z and G(z)
                l_generator, inter_layer_rct = self.discriminator(
                    self.noise_tensor, self.image_gen)

        # Loss Function Implementations
        with tf.name_scope("Loss_Functions"):
            # Discriminator
            # Discriminator sees the encoder result as true because it discriminates E(x), x as the real pair
            self.loss_dis_enc = tf.reduce_mean(
                tf.nn.sigmoid_cross_entropy_with_logits(
                    labels=self.true_labels, logits=l_encoder))
            self.loss_dis_gen = tf.reduce_mean(
                tf.nn.sigmoid_cross_entropy_with_logits(
                    labels=self.generated_labels, logits=l_generator))
            self.loss_discriminator = self.loss_dis_enc + self.loss_dis_gen

            if self.config.trainer.flip_labels:
                labels_gen = tf.zeros_like(l_generator)
                labels_enc = tf.ones_like(l_encoder)
            else:
                labels_gen = tf.ones_like(l_generator)
                labels_enc = tf.zeros_like(l_encoder)
            # Generator
            # Generator is considered as the true ones here because it tries to fool discriminator
            self.loss_generator = tf.reduce_mean(
                tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_gen,
                                                        logits=l_generator))
            # Encoder
            # Encoder is considered as the fake one because it tries to fool the discriminator also
            self.loss_encoder = tf.reduce_mean(
                tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_enc,
                                                        logits=l_encoder))
        # Optimizer Implementations
        with tf.name_scope("Optimizers"):
            # Build the optimizers
            self.generator_optimizer = tf.train.AdamOptimizer(
                self.config.trainer.generator_l_rate,
                beta1=self.config.trainer.optimizer_adam_beta1,
                beta2=self.config.trainer.optimizer_adam_beta2,
            )
            self.discriminator_optimizer = tf.train.AdamOptimizer(
                self.config.trainer.discriminator_l_rate,
                beta1=self.config.trainer.optimizer_adam_beta1,
                beta2=self.config.trainer.optimizer_adam_beta2,
            )
            self.encoder_optimizer = tf.train.AdamOptimizer(
                self.config.trainer.generator_l_rate,
                beta1=self.config.trainer.optimizer_adam_beta1,
                beta2=self.config.trainer.optimizer_adam_beta2,
            )
            # Collect all the variables
            all_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
            # Generator Network Variables
            self.generator_vars = [
                v for v in all_variables
                if v.name.startswith("BIGAN/Generator_Model")
            ]
            # Discriminator Network Variables
            self.discriminator_vars = [
                v for v in all_variables
                if v.name.startswith("BIGAN/Discriminator_Model")
            ]
            # Encoder Network Variables
            self.encoder_vars = [
                v for v in all_variables
                if v.name.startswith("BIGAN/Encoder_Model")
            ]
            # Create Training Operations
            # Generator Network Operations
            self.gen_update_ops = tf.get_collection(
                tf.GraphKeys.UPDATE_OPS, scope="BIGAN/Generator_Model")
            # Discriminator Network Operations
            self.disc_update_ops = tf.get_collection(
                tf.GraphKeys.UPDATE_OPS, scope="BIGAN/Discriminator_Model")
            # Encoder Network Operations
            self.enc_update_ops = tf.get_collection(
                tf.GraphKeys.UPDATE_OPS, scope="BIGAN/Encoder_Model")
            # Initialization of Optimizers
            with tf.control_dependencies(self.gen_update_ops):
                self.gen_op = self.generator_optimizer.minimize(
                    self.loss_generator,
                    var_list=self.generator_vars,
                    global_step=self.global_step_tensor,
                )
            with tf.control_dependencies(self.disc_update_ops):
                self.disc_op = self.discriminator_optimizer.minimize(
                    self.loss_discriminator, var_list=self.discriminator_vars)
            with tf.control_dependencies(self.enc_update_ops):
                self.enc_op = self.encoder_optimizer.minimize(
                    self.loss_encoder, var_list=self.encoder_vars)
            # Exponential Moving Average for Estimation
            self.dis_ema = tf.train.ExponentialMovingAverage(
                decay=self.config.trainer.ema_decay)
            maintain_averages_op_dis = self.dis_ema.apply(
                self.discriminator_vars)

            self.gen_ema = tf.train.ExponentialMovingAverage(
                decay=self.config.trainer.ema_decay)
            maintain_averages_op_gen = self.gen_ema.apply(self.generator_vars)

            self.enc_ema = tf.train.ExponentialMovingAverage(
                decay=self.config.trainer.ema_decay)
            maintain_averages_op_enc = self.enc_ema.apply(self.encoder_vars)

            with tf.control_dependencies([self.disc_op]):
                self.train_dis_op = tf.group(maintain_averages_op_dis)

            with tf.control_dependencies([self.gen_op]):
                self.train_gen_op = tf.group(maintain_averages_op_gen)

            with tf.control_dependencies([self.enc_op]):
                self.train_enc_op = tf.group(maintain_averages_op_enc)

        self.logger.info("Building Testing Graph...")

        with tf.variable_scope("BIGAN"):
            with tf.variable_scope("Encoder_Model"):
                self.noise_gen_ema = self.encoder(self.image_input,
                                                  getter=get_getter(
                                                      self.enc_ema))
            with tf.variable_scope("Generator_Model"):
                self.reconstruct_ema = self.generator(self.noise_gen_ema,
                                                      getter=get_getter(
                                                          self.gen_ema))
            with tf.variable_scope("Discriminator_Model"):
                self.l_encoder_ema, self.inter_layer_inp_ema = self.discriminator(
                    self.noise_gen_ema,  # E(x)
                    self.image_input,  # x
                    getter=get_getter(self.dis_ema),
                )
                self.l_generator_ema, self.inter_layer_rct_ema = self.discriminator(
                    self.noise_gen_ema,  # E(x)
                    self.reconstruct_ema,  # G(E(x))
                    getter=get_getter(self.dis_ema),
                )

        with tf.name_scope("Testing"):
            with tf.variable_scope("Reconstruction_Loss"):
                # LG(x) = ||x - G(E(x))||_1
                delta = self.image_input - self.reconstruct_ema
                delta_flat = tf.layers.Flatten()(delta)
                self.gen_score = tf.norm(
                    delta_flat,
                    ord=self.config.trainer.degree,
                    axis=1,
                    keepdims=False,
                    name="epsilon",
                )
            with tf.variable_scope("Discriminator_Loss"):
                if self.config.trainer.loss_method == "cross_e":
                    self.dis_score = tf.nn.sigmoid_cross_entropy_with_logits(
                        labels=tf.ones_like(self.l_encoder_ema),
                        logits=self.l_encoder_ema)
                elif self.config.trainer.loss_method == "fm":
                    fm = self.inter_layer_inp_ema - self.inter_layer_rct_ema
                    fm = tf.layers.Flatten()(fm)
                    self.dis_score = tf.norm(fm,
                                             ord=self.config.trainer.degree,
                                             axis=1,
                                             keepdims=False,
                                             name="d_loss")
                self.dis_score = tf.squeeze(self.dis_score)
            with tf.variable_scope("Score"):
                self.list_scores = (
                    1 - self.config.trainer.weight
                ) * self.dis_score + self.config.trainer.weight * self.gen_score

        if self.config.trainer.enable_early_stop:
            self.rec_error_valid = tf.reduce_mean(self.list_scores)

        if self.config.log.enable_summary:
            with tf.name_scope("Summary"):
                with tf.name_scope("Disc_Summary"):
                    tf.summary.scalar("loss_discriminator",
                                      self.loss_discriminator, ["dis"])
                    tf.summary.scalar("loss_dis_encoder", self.loss_dis_enc,
                                      ["dis"])
                    tf.summary.scalar("loss_dis_gen", self.loss_dis_gen,
                                      ["dis"])
                with tf.name_scope("Gen_Summary"):
                    tf.summary.scalar("loss_generator", self.loss_generator,
                                      ["gen"])
                    tf.summary.scalar("loss_encoder", self.loss_encoder,
                                      ["gen"])
                with tf.name_scope("Image_Summary"):
                    tf.summary.image("reconstruct", self.reconstructed, 3,
                                     ["image"])
                    tf.summary.image("input_images", self.image_input, 3,
                                     ["image"])
        if self.config.trainer.enable_early_stop:
            with tf.name_scope("validation_summary"):
                tf.summary.scalar("valid", self.rec_error_valid, ["v"])

        self.sum_op_dis = tf.summary.merge_all("dis")
        self.sum_op_gen = tf.summary.merge_all("gen")
        self.sum_op_im = tf.summary.merge_all("image")
        self.sum_op_valid = tf.summary.merge_all("v")