Example #1
0
  def discriminator(self, image, y=None, reuse=False):
    with tf.variable_scope("discriminator") as scope:
      if reuse:
        scope.reuse_variables()

      if not self.y_dim:
        h0 = lrelu(conv2d(image, self.df_dim, name='d_h0_conv'))
        h1 = lrelu(self.d_bn1(conv2d(h0, self.df_dim*2, name='d_h1_conv')))
        h2 = lrelu(self.d_bn2(conv2d(h1, self.df_dim*4, name='d_h2_conv')))
        h3 = lrelu(self.d_bn3(conv2d(h2, self.df_dim*8, name='d_h3_conv')))
        h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1, 'd_h3_lin')

        return tf.nn.sigmoid(h4), h4
      else:
        yb = tf.reshape(y, [self.batch_size, 1, 1, self.y_dim])
        x = conv_cond_concat(image, yb)

        h0 = lrelu(conv2d(x, self.c_dim + self.y_dim, name='d_h0_conv'))
        h0 = conv_cond_concat(h0, yb)

        h1 = lrelu(self.d_bn1(conv2d(h0, self.df_dim + self.y_dim, name='d_h1_conv')))
        h1 = tf.reshape(h1, [self.batch_size, -1])
        h1 = tf.concat_v2([h1, y], 1)

        h2 = lrelu(self.d_bn2(linear(h1, self.dfc_dim, 'd_h2_lin')))
        h2 = tf.concat_v2([h2, y], 1)

        h3 = linear(h2, 1, 'd_h3_lin')

        return tf.nn.sigmoid(h3), h3
  def _testConfMatrixOnTensors(self, tf_dtype, np_dtype):
    with self.test_session() as sess:
      m_neg = tf.placeholder(dtype=tf.float32)
      m_pos = tf.placeholder(dtype=tf.float32)
      s = tf.placeholder(dtype=tf.float32)

      neg = tf.random_normal([20], mean=m_neg, stddev=s, dtype=tf.float32)
      pos = tf.random_normal([20], mean=m_pos, stddev=s, dtype=tf.float32)

      data = tf.concat_v2([neg, pos], 0)
      data = tf.cast(tf.round(data), tf_dtype)
      data = tf.minimum(tf.maximum(data, 0), 1)
      lab = tf.concat_v2(
          [tf.zeros(
              [20], dtype=tf_dtype), tf.ones(
                  [20], dtype=tf_dtype)], 0)

      cm = tf.confusion_matrix(
          lab, data, dtype=tf_dtype, num_classes=2)

      d, l, cm_out = sess.run([data, lab, cm], {m_neg: 0.0,
                                                m_pos: 1.0,
                                                s: 1.0})

      truth = np.zeros([2, 2], dtype=np_dtype)
      try:
        range_builder = xrange
      except NameError:  # In Python 3.
        range_builder = range
      for i in range_builder(len(d)):
        truth[d[i], l[i]] += 1

      self.assertEqual(cm_out.dtype, np_dtype)
      self.assertAllClose(cm_out, truth, atol=1e-10)
Example #3
0
    def __call__(self, input_tuple, state, scope=None):
        with vs.variable_scope(scope or type(self).__name__):
            self._init_parameters()

            _input, target, dInput_dF = input_tuple
            
            u, s, r, inner_spikes, dW, dF, reward, reward_mean = state
            
            s = (1.0 - tau_syn) * s + tf.concat_v2([_input, inner_spikes], 1)
            u = (1.0 - tau_mem) * u + mo.matmul(s, self.W)
            r = (1.0 - tau_refr) * r
            

            act_raw = self._activation(u)
            act = act_raw * tf.exp(-r)
            
            spikes = tf.where(
                act > tf.random_uniform([batch_size, self._num_units]),
                tf.ones([batch_size, self._num_units]),
                tf.zeros([batch_size, self._num_units])
            )   
            
            hidden_spikes, _ = self.slice(spikes)
            _, act_visible = self.slice(act)
            
            
            reward_mean = (1.0 - tau_long) * reward_mean + tau_long * reward
            reward = (1.0 - tau_learn) * reward + tau_learn * tf.reduce_sum(
                target * safe_log(act_visible*dt) + (1.0 - target) * safe_log(1.0 - act_visible*dt)
            , 1)
            
            r += spikes * amp_refr

            act_grad = tf.gradients([act], [u])[0]
            
            learn_target = tf.concat_v2([hidden_spikes, target], 1)

            factor = tf.concat_v2([
                (reward - reward_mean) * tf.ones((batch_size, hidden_size,)), 
                tf.ones((batch_size, visible_size,))
            ], 1)
            
            neuron_derivative = tf.reduce_sum( (act_grad/act_raw) * (learn_target - act) * factor, 0)
            

            Wsliced = tf.slice(self.W, [0, 0], [self._filters_num, self._num_units])
            
            dF_deriv_part = tf.squeeze(mo.matmul(Wsliced, tf.expand_dims(neuron_derivative, 1)))
            
            
            dW += lrate * outer(
                tf.reduce_sum(s, 0),
                neuron_derivative
            )
            
            dInput_dF = tf.reduce_mean(dInput_dF, 0)
            
            dF += lrate * dF_deriv_part * dInput_dF
            
            return GLMOutputTuple(spikes, act, factor, reward, reward_mean), GLMStateTuple(u, s, r, spikes, dW, dF, reward, reward_mean)
Example #4
0
 def testAttentionCellWrapperCorrectResult(self):
   num_units = 4
   attn_length = 6
   batch_size = 2
   expected_output = np.array(
       [[0.955392, 0.408507, -0.60122, 0.270718],
        [0.903681, 0.331165, -0.500238, 0.224052]],
       dtype=np.float32)
   expected_state = np.array(
       [[
           0.81331915, 0.32036272, 0.28079176, 1.08888793, 0.41264394,
           0.1062041, 0.10444493, 0.32050529, 0.64655536, 0.70794445,
           0.51896095, 0.31809306, 0.58086717, 0.49446869, 0.7641536,
           0.12814975, 0.92231739, 0.89857256, 0.21889746, 0.38442063,
           0.53481543, 0.8876909, 0.45823169, 0.5905602, 0.78038228,
           0.56501579, 0.03971386, 0.09870267, 0.8074435, 0.66821432,
           0.99211812, 0.12295902, 1.01412082, 0.33123279, -0.71114945,
           0.40583119
       ], [
           0.59962207, 0.42597458, -0.22491696, 0.98063421, 0.32548007,
           0.11623692, -0.10100613, 0.27708149, 0.76956916, 0.6360054,
           0.51719815, 0.50458527, 0.73000264, 0.66986895, 0.73576689,
           0.86301267, 0.87887371, 0.35185754, 0.93417215, 0.64732957,
           0.63173044, 0.66627824, 0.53644657, 0.20477486, 0.98458421,
           0.38277245, 0.03746676, 0.92510188, 0.57714164, 0.84932971,
           0.36127412, 0.12125921, 0.99780077, 0.31886846, -0.67595094,
           0.56531656
       ]],
       dtype=np.float32)
   seed = 12345
   tf.set_random_seed(seed)
   for state_is_tuple in [False, True]:
     with tf.Session() as sess:
       with tf.variable_scope("state_is_tuple", reuse=state_is_tuple):
         lstm_cell = tf.contrib.rnn.BasicLSTMCell(
             num_units, state_is_tuple=state_is_tuple)
         cell = tf.contrib.rnn.AttentionCellWrapper(
             lstm_cell, attn_length, state_is_tuple=state_is_tuple)
         zeros1 = tf.random_uniform(
             (batch_size, num_units), 0.0, 1.0, seed=seed + 1)
         zeros2 = tf.random_uniform(
             (batch_size, num_units), 0.0, 1.0, seed=seed + 2)
         zeros3 = tf.random_uniform(
             (batch_size, num_units), 0.0, 1.0, seed=seed + 3)
         attn_state_zeros = tf.random_uniform(
             (batch_size, attn_length * num_units), 0.0, 1.0, seed=seed + 4)
         zero_state = ((zeros1, zeros2), zeros3, attn_state_zeros)
         if not state_is_tuple:
           zero_state = tf.concat_v2([
               zero_state[0][0], zero_state[0][1], zero_state[1], zero_state[2]
           ], 1)
         inputs = tf.random_uniform(
             (batch_size, num_units), 0.0, 1.0, seed=seed + 5)
         output, state = cell(inputs, zero_state)
         if state_is_tuple:
           state = tf.concat_v2([state[0][0], state[0][1], state[1], state[2]],
                                1)
         sess.run(tf.global_variables_initializer())
         self.assertAllClose(sess.run(output), expected_output)
         self.assertAllClose(sess.run(state), expected_state)
Example #5
0
    def _define_partial_maximization_operation(self, shard_id, shard):
        """Computes the partial statistics of the means and covariances.

    Args:
      shard_id: current shard id.
      shard: current data shard, 1 X num_examples X dimensions.
    """
        # Soft assignment of each data point to each of the two clusters.
        self._points_in_k[shard_id] = tf.reduce_sum(self._w[shard_id],
                                                    0,
                                                    keep_dims=True)
        # Partial means.
        w_mul_x = tf.expand_dims(
            tf.matmul(self._w[shard_id],
                      tf.squeeze(shard, [0]),
                      transpose_a=True), 1)
        self._w_mul_x.append(w_mul_x)
        # Partial covariances.
        x = tf.concat_v2([shard for _ in range(self._num_classes)], 0)
        x_trans = tf.transpose(x, perm=[0, 2, 1])
        x_mul_w = tf.concat_v2([
            tf.expand_dims(x_trans[k, :, :] * self._w[shard_id][:, k], 0)
            for k in range(self._num_classes)
        ], 0)
        self._w_mul_x2.append(tf.matmul(x_mul_w, x))
 def predict_mean_and_var(self, Fmu, Fvar):
     mu_list, var_list = zip(*[
         lik.predict_mean_and_var(Fmu, Fvar) for lik in self.likelihood_list
     ])
     mu = tf.concat_v2(mu_list, 1)
     var = tf.concat_v2(var_list, 1)
     return mu, var
Example #7
0
  def _define_distance_to_clusters(self, data):
    """Defines the Mahalanobis distance to the assigned Gaussian."""
    # TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input -
    # mean) from log probability function.
    self._all_scores = []
    for shard in data:
      all_scores = []
      shard = tf.expand_dims(shard, 0)
      for c in xrange(self._num_classes):
        if self._covariance_type == FULL_COVARIANCE:
          cov = self._covs[c, :, :]
        elif self._covariance_type == DIAG_COVARIANCE:
          cov = tf.diag(self._covs[c, :])
        inverse = tf.matrix_inverse(cov + self._min_var)
        inv_cov = tf.tile(
            tf.expand_dims(inverse, 0), tf.stack([self._num_examples, 1, 1]))
        diff = tf.transpose(shard - self._means[c, :, :], perm=[1, 0, 2])
        m_left = tf.matmul(diff, inv_cov)
        all_scores.append(
            tf.sqrt(tf.matmul(
                m_left, tf.transpose(
                    diff, perm=[0, 2, 1]))))
      self._all_scores.append(
          tf.reshape(
              tf.concat_v2(all_scores, 1),
              tf.stack([self._num_examples, self._num_classes])))

    # Distance to the associated class.
    self._all_scores = tf.concat_v2(self._all_scores, 0)
    assignments = tf.concat_v2(self.assignments(), 0)
    rows = tf.to_int64(tf.range(0, self._num_examples))
    indices = tf.concat_v2(
        [tf.expand_dims(rows, 1), tf.expand_dims(assignments, 1)], 1)
    self._scores = tf.gather_nd(self._all_scores, indices)
        def partRotator(self, images, name, batch_size=10, reuse=False):
        #HW 40x40, 32x40, 32x48
        with tf.variable_scope(name) as scope:
            if reuse:
                scope.reuse_variables()
            c0 = lrelu(conv2d(images, self.gf_dim, d_h=1, d_w=1, name="p_conv0"))
            c0r = resblock(c0, name="p_conv0_res")
            c1 = lrelu(batch_norm(conv2d(c0r, self.gf_dim*2, name="p_conv1"),name="p_bnc1"))
            #down1
            c1r = resblock(c1, name="p_conv1_res")
            c2 = lrelu(batch_norm(conv2d(c1r, self.gf_dim*4, name='p_conv2'),name="p_bnc2"))
            #down2
            c2r = resblock(c2, name="p_conv2_res")
            c3 = lrelu(batch_norm(conv2d(c2r, self.gf_dim*8, name='p_conv3'),name="p_bnc3"))
            #down3 5x5, 4x5, 4x6
            c3r = resblock(c3, name="p_conv3_res")
            c3r2 = resblock(c3r, name="p_conv3_res2")

            shape = c3r2.get_shape().as_list()
            d1 = lrelu(batch_norm(deconv2d(c3r2, [shape[0], shape[1] * 2, shape[2] * 2, self.gf_dim*4], name="p_deconv1"), name="p_bnd1"))
            #up1
            after_select_d1 = lrelu(batch_norm(conv2d(tf.concat_v2([d1, c2r], axis=3), self.gf_dim*4, d_h=1, d_w=1, name="p_deconv1_s"),name="p_bnd1_s"))
            d1_r = resblock(after_select_d1, name="p_deconv1_res")
            d2 = lrelu(batch_norm(deconv2d(d1_r, [shape[0], shape[1] * 4, shape[2] * 4, self.gf_dim*2], name="p_deconv2"), name="p_bnd2"))
            #up2
            after_select_d2 = lrelu(batch_norm(conv2d(tf.concat_v2([d2, c1r], axis=3), self.gf_dim*2, d_h=1, d_w=1, name="p_deconv2_s"),name="p_bnd2_s"))
            d2_r = resblock(after_select_d2, name="p_deconv2_res")
            d3 = lrelu(batch_norm(deconv2d(d2_r, [shape[0], shape[1] * 8, shape[2] * 8, self.gf_dim], name="p_deconv3"), name="p_bnd3"))
            #up3
            after_select_d3 = lrelu(batch_norm(conv2d(tf.concat_v2([d3, c0r], axis=3), self.gf_dim, d_h=1, d_w=1, name="p_deconv3_s"),name="p_bnd3_s"))
            d3_r = resblock(after_select_d3, name="p_deconv3_res")

            check_part = tf.nn.tanh(conv2d(d3_r, 3, d_h=1, d_w=1, name="p_check"))

        return d3_r, check_part
Example #9
0
    def _define_distance_to_clusters(self, data):
        """Defines the Mahalanobis distance to the assigned Gaussian."""
        # TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input -
        # mean) from log probability function.
        self._all_scores = []
        for shard in data:
            all_scores = []
            shard = tf.expand_dims(shard, 0)
            for c in xrange(self._num_classes):
                if self._covariance_type == FULL_COVARIANCE:
                    cov = self._covs[c, :, :]
                elif self._covariance_type == DIAG_COVARIANCE:
                    cov = tf.diag(self._covs[c, :])
                inverse = tf.matrix_inverse(cov + self._min_var)
                inv_cov = tf.tile(tf.expand_dims(inverse, 0),
                                  tf.stack([self._num_examples, 1, 1]))
                diff = tf.transpose(shard - self._means[c, :, :],
                                    perm=[1, 0, 2])
                m_left = tf.matmul(diff, inv_cov)
                all_scores.append(
                    tf.sqrt(
                        tf.matmul(m_left, tf.transpose(diff, perm=[0, 2, 1]))))
            self._all_scores.append(
                tf.reshape(tf.concat_v2(all_scores, 1),
                           tf.stack([self._num_examples, self._num_classes])))

        # Distance to the associated class.
        self._all_scores = tf.concat_v2(self._all_scores, 0)
        assignments = tf.concat_v2(self.assignments(), 0)
        rows = tf.to_int64(tf.range(0, self._num_examples))
        indices = tf.concat_v2(
            [tf.expand_dims(rows, 1),
             tf.expand_dims(assignments, 1)], 1)
        self._scores = tf.gather_nd(self._all_scores, indices)
def symmetric_feedforward_weights(weights):
	sizes = [weights[0].get_shape()[0].value]
	sizes += [ w.get_shape()[0].value for w in weights[1:] ]
	sizes += [weights[-1].get_shape()[1].value]

	res = []
	for pre_id, pre_size in enumerate(sizes):
		stack = []
		for post_id, post_size in enumerate(sizes):
			if pre_id == post_id - 1 and pre_id < len(weights):
				stack.append(weights[pre_id])
			elif post_id == pre_id -1 and post_id < len(weights):
				stack.append(tf.transpose(weights[post_id]))
			else:
				pre_zeros = tf.zeros((pre_size, post_size))
				stack.append(pre_zeros)
		
		res.append(tf.concat_v2(stack, 1))


	res = tf.concat_v2(res, 0)

	# for i in xrange(res_v.shape[0]):
	# 	for j in xrange(res_v.shape[1]):
	# 		assert res_v[i, j] == res_v[j, i]

	return res
def symmetric_feedforward_weights(weights):
    sizes = [weights[0].get_shape()[0].value]
    sizes += [w.get_shape()[0].value for w in weights[1:]]
    sizes += [weights[-1].get_shape()[1].value]

    res = []
    for pre_id, pre_size in enumerate(sizes):
        stack = []
        for post_id, post_size in enumerate(sizes):
            if pre_id == post_id - 1 and pre_id < len(weights):
                stack.append(weights[pre_id])
            elif post_id == pre_id - 1 and post_id < len(weights):
                stack.append(tf.transpose(weights[post_id]))
            else:
                pre_zeros = tf.zeros((pre_size, post_size))
                stack.append(pre_zeros)

        res.append(tf.concat_v2(stack, 1))

    res = tf.concat_v2(res, 0)

    # for i in xrange(res_v.shape[0]):
    # 	for j in xrange(res_v.shape[1]):
    # 		assert res_v[i, j] == res_v[j, i]

    return res
Example #12
0
        def calcLoss(targetss, logitss, batch_sizee, num_sampless, num_stepss,
                     indices):
            indices2 = tf.zeros([num_sampless], tf.int32)
            print("indices2")
            print(indices2)

            #heightt=tf.to_int32(tf.shape(logitss)[0]);
            heightt = logitss.get_shape().as_list()[0]
            print("heightt")
            print(heightt)
            for i in range(1, heightt):
                #print("Loop Beginsssss");
                tempp = tf.fill([num_sampless], i)
                indices2 = tf.concat_v2([indices2, tempp], 0)

            print("indices2")
            print(indices2)
            indices = indices2 + tf.reshape(indices, [-1])
            print("indices")
            print(indices)
            negative_logitss = tf.gather(tf.reshape(logitss, [-1]), indices)
            negative_logitss = tf.reshape(negative_logitss,
                                          [heightt, num_sampless])

            print("logitss")
            print(logitss)
            print("negative_logitss")
            print(negative_logitss)
            print("indices")
            print(indices)
            print("targetss")
            print(targetss)
            targetss_flattened = tf.range(
                0,
                tf.shape(logitss)[0]) * tf.shape(logitss)[1] + targetss
            print("targetss_flattened")
            print(targetss_flattened)
            true_logitss = tf.gather(tf.reshape(logitss, [-1]),
                                     targetss_flattened)
            print("true_logitss")
            print(true_logitss)
            new_targetss = tf.zeros([logitss.get_shape().as_list()[0]],
                                    tf.int32)
            print("new_targetss")
            print(new_targetss)
            true_logitss_2d = tf.reshape(true_logitss,
                                         [tf.shape(true_logitss)[0], 1])
            print("true_logitss_2d")
            print(true_logitss_2d)

            print("new_logitss")
            new_logitss = tf.concat_v2([true_logitss_2d, negative_logitss], 1)
            print(new_logitss)

            loss = tf.nn.seq2seq.sequence_loss_by_example(
                [new_logitss], [new_targetss],
                [tf.ones([batch_sizee * num_stepss], dtype=data_type())])

            return loss
Example #13
0
def boston_eval_fn():
  boston = tf.contrib.learn.datasets.load_boston()
  n_examples = len(boston.target)
  features = tf.reshape(
      tf.constant(boston.data), [n_examples, _BOSTON_INPUT_DIM])
  labels = tf.reshape(tf.constant(boston.target), [n_examples, 1])
  return tf.concat_v2([features, features], 0), tf.concat_v2([labels, labels],
                                                             0)
Example #14
0
def boston_eval_fn():
    boston = tf.contrib.learn.datasets.load_boston()
    n_examples = len(boston.target)
    features = tf.reshape(tf.constant(boston.data),
                          [n_examples, _BOSTON_INPUT_DIM])
    labels = tf.reshape(tf.constant(boston.target), [n_examples, 1])
    return tf.concat_v2([features, features],
                        0), tf.concat_v2([labels, labels], 0)
Example #15
0
  def sampler(self, z, y=None):
    with tf.variable_scope("generator") as scope:
      scope.reuse_variables()

      if not self.y_dim:
        s_h, s_w = self.output_height, self.output_width
        s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
        s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
        s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
        s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)

        # project `z` and reshape
        h0 = tf.reshape(
            linear(z, self.gf_dim*8*s_h16*s_w16, 'g_h0_lin'),
            [-1, s_h16, s_w16, self.gf_dim * 8])
        h0 = tf.nn.relu(self.g_bn0(h0, train=False))

        h1 = deconv2d(h0, [self.batch_size, s_h8, s_w8, self.gf_dim*4], name='g_h1')
        h1 = tf.nn.relu(self.g_bn1(h1, train=False))

        h2 = deconv2d(h1, [self.batch_size, s_h4, s_w4, self.gf_dim*2], name='g_h2')
        h2 = tf.nn.relu(self.g_bn2(h2, train=False))

        h3 = deconv2d(h2, [self.batch_size, s_h2, s_w2, self.gf_dim*1], name='g_h3')
        h3 = tf.nn.relu(self.g_bn3(h3, train=False))

        h4 = deconv2d(h3, [self.batch_size, s_h, s_w, self.c_dim], name='g_h4')

        return tf.nn.tanh(h4)
      else:
        s_h, s_w = self.output_height, self.output_width
        s_h2, s_h4 = int(s_h/2), int(s_h/4)
        s_w2, s_w4 = int(s_w/2), int(s_w/4)

        # yb = tf.reshape(y, [-1, 1, 1, self.y_dim])
        yb = tf.reshape(y, [self.batch_size, 1, 1, self.y_dim])
        try:
          z = tf.concat_v2([z, y], 1)
        catch:
          z = tf.concat([z, y], 1)

        h0 = tf.nn.relu(self.g_bn0(linear(z, self.gfc_dim, 'g_h0_lin')))
        try:
          h0 = tf.concat_v2([h0, y], 1)
        catch:
          h0 = tf.concat([h0, y], 1)
        

        h1 = tf.nn.relu(self.g_bn1(
            linear(h0, self.gf_dim*2*s_h4*s_w4, 'g_h1_lin'), train=False))
        h1 = tf.reshape(h1, [self.batch_size, s_h4, s_w4, self.gf_dim * 2])
        h1 = conv_cond_concat(h1, yb)

        h2 = tf.nn.relu(self.g_bn2(
            deconv2d(h1, [self.batch_size, s_h2, s_w2, self.gf_dim * 2], name='g_h2'), train=False))
        h2 = conv_cond_concat(h2, yb)

        return tf.nn.sigmoid(deconv2d(h2, [self.batch_size, s_h, s_w, self.c_dim], name='g_h3'))
def VI_Block(X, S1, S2, config):
    k = config.k  # Number of value iterations performed
    ch_i = config.ch_i  # Channels in input layer
    ch_h = config.ch_h  # Channels in initial hidden layer
    ch_q = config.ch_q  # Channels in q layer (~actions)
    state_batch_size = config.statebatchsize  # k+1 state inputs for each channel

    bias = tf.Variable(np.random.randn(1, 1, 1, ch_h) * 0.01, dtype=tf.float32)
    # weights from inputs to q layer (~reward in Bellman equation)
    w0 = tf.Variable(np.random.randn(3, 3, ch_i, ch_h) * 0.01,
                     dtype=tf.float32)
    w1 = tf.Variable(np.random.randn(1, 1, ch_h, 1) * 0.01, dtype=tf.float32)
    w = tf.Variable(np.random.randn(3, 3, 1, ch_q) * 0.01, dtype=tf.float32)
    # feedback weights from v layer into q layer (~transition probabilities in Bellman equation)
    w_fb = tf.Variable(np.random.randn(3, 3, 1, ch_q) * 0.01, dtype=tf.float32)
    w_o = tf.Variable(np.random.randn(ch_q, 8) * 0.01, dtype=tf.float32)

    # initial conv layer over image+reward prior
    h = conv2d_flipkernel(X, w0, name="h0") + bias

    r = conv2d_flipkernel(h, w1, name="r")
    q = conv2d_flipkernel(r, w, name="q")
    v = tf.reduce_max(q, axis=3, keep_dims=True, name="v")

    for i in range(0, k - 1):
        rv = tf.concat_v2([r, v], 3)
        wwfb = tf.concat_v2([w, w_fb], 2)

        q = conv2d_flipkernel(rv, wwfb, name="q")
        v = tf.reduce_max(q, axis=3, keep_dims=True, name="v")

    # do one last convolution
    q = conv2d_flipkernel(tf.concat_v2([r, v], 3),
                          tf.concat_v2([w, w_fb], 2),
                          name="q")

    # CHANGE TO THEANO ORDERING
    # Since we are selecting over channels, it becomes easier to work with
    # the tensor when it is in NCHW format vs NHWC
    q = tf.transpose(q, perm=[0, 3, 1, 2])

    # Select the conv-net channels at the state position (S1,S2).
    # This intuitively corresponds to each channel representing an action, and the convnet the Q function.
    # The tricky thing is we want to select the same (S1,S2) position *for each* channel and for each sample
    # TODO: performance can be improved here by substituting expensive
    #       transpose calls with better indexing for gather_nd
    bs = tf.shape(q)[0]
    rprn = tf.reshape(
        tf.tile(tf.reshape(tf.range(bs), [-1, 1]), [1, state_batch_size]),
        [-1])
    ins1 = tf.cast(tf.reshape(S1, [-1]), tf.int32)
    ins2 = tf.cast(tf.reshape(S2, [-1]), tf.int32)
    idx_in = tf.transpose(tf.stack([ins1, ins2, rprn]), [1, 0])
    q_out = tf.gather_nd(tf.transpose(q, [2, 3, 0, 1]), idx_in, name="q_out")

    # softmax output weights
    output = tf.nn.softmax(tf.matmul(q_out, w_o), name="output")
    return output
Example #17
0
 def refresh_shortlist():
     """Update the shortlist with the highest scores in id_to_score."""
     new_scores, new_ids = tf.nn.top_k(self.id_to_score, self.shortlist_size)
     smallest_new_score = tf.reduce_min(new_scores)
     new_length = tf.reduce_sum(tf.to_int32(tf.greater(new_scores, tf.float32.min)))
     u1 = self.sl_ids.assign(tf.to_int64(tf.concat_v2([[new_length], new_ids], 0)))
     u2 = self.sl_scores.assign(tf.concat_v2([[smallest_new_score], new_scores], 0))
     self.last_ops = [u1, u2]
     return tf.group(u1, u2)
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
    correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
    correctly_recognised_indices = tf.squeeze(
        tf.where(correct_prediction),
        [1])  # indices of correctly recognised images
    incorrectly_recognised_indices = tf.squeeze(
        tf.where(tf.logical_not(correct_prediction)),
        [1])  # indices of incorrectly recognised images
    everything_incorrect_first = tf.concat_v2(
        [incorrectly_recognised_indices, correctly_recognised_indices],
        0)  # images reordered with indeces of unrecognised images first
    everything_incorrect_first = tf.slice(
        everything_incorrect_first, [0],
        [n])  # compute first 100 only - no space to display more anyway
    # compute n=100 digits to display only
    Xs = tf.gather(X, everything_incorrect_first)
    Ys = tf.gather(Y, everything_incorrect_first)
    Ys_ = tf.gather(Y_, everything_incorrect_first)
    correct_prediction_s = tf.gather(correct_prediction,
                                     everything_incorrect_first)

    digits_left = tf.image.grayscale_to_rgb(
        tensorflowvisu_digits.digits_left())
    correct_tags = tf.gather(digits_left, tf.argmax(
        Ys_, 1))  # correct digits to be printed on the images
    digits_right = tf.image.grayscale_to_rgb(
        tensorflowvisu_digits.digits_right())
    computed_tags = tf.gather(digits_right, tf.argmax(
        Ys, 1))  # computed digits to be printed on the images
    #superimposed_digits = correct_tags+computed_tags
    superimposed_digits = tf.where(
        correct_prediction_s, tf.zeros_like(correct_tags),
        correct_tags + computed_tags
    )  # only pring the correct and computed digits on unrecognised images
    correct_bkg = tf.reshape(tf.tile([1.3, 1.3, 1.3], [28 * 28]),
                             [1, 28, 28, 3])  # white background
    incorrect_bkg = tf.reshape(tf.tile([1.3, 1.0, 1.0], [28 * 28]),
                               [1, 28, 28, 3])  # red background
    recognised_bkg = tf.gather(
        tf.concat_v2([incorrect_bkg, correct_bkg], 0),
        tf.cast(correct_prediction_s, tf.int32)
    )  # pick either the red or the white background depending on recognised status

    I = tf.image.grayscale_to_rgb(Xs)
    I = (
        (1 - (I + superimposed_digits)) * recognised_bkg
    ) / 1.3  # stencil extra data on top of images and reorder them unrecognised first
    I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
    Islices = []  # 100 images => 10x10 image block
    for imslice in range(lines):
        Islices.append(
            tf.concat_v2(
                tf.unstack(
                    tf.slice(I, [imslice * n // lines, 0, 0, 0],
                             [n // lines, 28, 28, 3])), 1))
    I = tf.concat_v2(Islices, 0)
    return I
Example #19
0
  def build_model(self):
    sc = predictron_arg_scope()
    with tf.variable_scope('state'):
      with slim.arg_scope(sc):
        state = slim.conv2d(self.inputs, 32, [3, 3], scope='conv1')
        state = layers.batch_norm(state, activation_fn=tf.nn.relu, scope='conv1/preact')
        state = slim.conv2d(state, 32, [3, 3], scope='conv2')
        state = layers.batch_norm(state, activation_fn=tf.nn.relu, scope='conv2/preact')

    iter_template = tf.make_template('iter', self.iter_func, unique_name_='iter')

    rewards_arr = []
    gammas_arr = []
    lambdas_arr = []
    values_arr = []

    for k in range(self.max_depth):
      state, reward, gamma, lambda_, value = iter_template(state)
      rewards_arr.append(reward)
      gammas_arr.append(gamma)
      lambdas_arr.append(lambda_)
      values_arr.append(value)

    _, _, _, _, value = iter_template(state)
    # K + 1 elements
    values_arr.append(value)

    bs = tf.shape(self.inputs)[0]
    # [batch_size, K * maze_size]
    self.rewards = tf.pack(rewards_arr, axis=1)
    # [batch_size, K, maze_size]
    self.rewards = tf.reshape(self.rewards, [bs, self.max_depth, self.maze_size])
    # [batch_size, K + 1, maze_size]
    self.rewards = tf.concat_v2(values=[tf.zeros(shape=[bs, 1, self.maze_size], dtype=tf.float32), self.rewards],
                                axis=1, name='rewards')

    # [batch_size, K * maze_size]
    self.gammas = tf.pack(gammas_arr, axis=1)
    # [batch_size, K, maze_size]
    self.gammas = tf.reshape(self.gammas, [bs, self.max_depth, self.maze_size])
    # [batch_size, K + 1, maze_size]
    self.gammas = tf.concat_v2(values=[tf.ones(shape=[bs, 1, self.maze_size], dtype=tf.float32), self.gammas],
                               axis=1, name='gammas')

    # [batch_size, K * maze_size]
    self.lambdas = tf.pack(lambdas_arr, axis=1)
    # [batch_size, K, maze_size]
    self.lambdas = tf.reshape(self.lambdas, [-1, self.max_depth, self.maze_size])

    # [batch_size, (K + 1) * maze_size]
    self.values = tf.pack(values_arr, axis=1)
    # [batch_size, K + 1, maze_size]
    self.values = tf.reshape(self.values, [-1, (self.max_depth + 1), self.maze_size])

    self.build_preturns()
    self.build_lambda_preturns()
Example #20
0
def _phase_shift(I, r):
    bsize, a, b, c = I.get_shape().as_list()
    bsize = tf.shape(I)[0] # Handling Dimension(None) type for undefined batch dim
    X = tf.reshape(I, (bsize, a, b, r, r))
    X = tf.transpose(X, (0, 1, 2, 4, 3))  # bsize, a, b, 1, 1
    X = tf.split(X, a, 1)  # a, [bsize, b, r, r]
    X = tf.concat_v2([tf.squeeze(x, axis=1) for x in X], 2)  # bsize, b, a*r, r
    X = tf.split(X, b, 1)  # b, [bsize, a*r, r]
    X = tf.concat_v2([tf.squeeze(x, axis=1) for x in X], 2)  # bsize, a*r, b*r
    return tf.reshape(X, (bsize, a*r, b*r, 1))
  def testConcat(self):
    tf_val = tf.concat_v2([[16, 37], tf.placeholder(tf.int32, shape=(2,))], 0)
    c_val = tensor_util.constant_value_as_shape(tf_val)
    self.assertEqual([16, 37, None, None], c_val.as_list())

    tf_val = tf.concat_v2(
        [[16, 37], tf.placeholder(
            tf.int32, shape=(1,)), [48]], 0)
    c_val = tensor_util.constant_value_as_shape(tf_val)
    self.assertEqual([16, 37, None, 48], c_val.as_list())
Example #22
0
    def testConcat(self):
        tf_val = tf.concat_v2(
            [[16, 37], tf.placeholder(tf.int32, shape=(2, ))], 0)
        c_val = tensor_util.constant_value_as_shape(tf_val)
        self.assertEqual([16, 37, None, None], c_val.as_list())

        tf_val = tf.concat_v2(
            [[16, 37], tf.placeholder(tf.int32, shape=(1, )), [48]], 0)
        c_val = tensor_util.constant_value_as_shape(tf_val)
        self.assertEqual([16, 37, None, 48], c_val.as_list())
Example #23
0
 def forward(self, x, z):  #, h_tm1, pz_tm1):
     xz = tf.concat_v2([x, tf.expand_dims(z, -1)], 2)
     h0 = tf.zeros((1, tf.shape(x)[1], self.h_size))
     h = self.rcnn.forward(xz)
     h_prev = tf.concat_v2([h0, h[:-1]], 0)
     pz = tf.nn.sigmoid(
         tf.matmul(tf.reshape(x, [-1, self.in_size]), self.w1) +
         tf.matmul(tf.reshape(h_prev, [-1, self.h_size]), self.w2) +
         self.bias)
     return tf.reshape(pz, [tf.shape(x)[0], tf.shape(x)[1]])
def unpool(x, size):
    out = tf.concat_v2([x, tf.zeros_like(x)], 3)
    out = tf.concat_v2([out, tf.zeros_like(out)], 2)

    sh = x.get_shape().as_list()
    if None not in sh[1:]:
        out_size = [-1, sh[1] * size, sh[2] * size, sh[3]]
        return tf.reshape(out, out_size)

    shv = tf.shape(x)
    ret = tf.reshape(out, tf.stack([-1, shv[1] * size, shv[2] * size, sh[3]]))
    ret.set_shape([None, None, None, sh[3]])
    return ret
Example #25
0
 def refresh_shortlist():
     """Update the shortlist with the highest scores in id_to_score."""
     new_scores, new_ids = tf.nn.top_k(self.id_to_score,
                                       self.shortlist_size)
     smallest_new_score = tf.reduce_min(new_scores)
     new_length = tf.reduce_sum(
         tf.to_int32(tf.greater(new_scores, tf.float32.min)))
     u1 = self.sl_ids.assign(
         tf.to_int64(tf.concat_v2([[new_length], new_ids], 0)))
     u2 = self.sl_scores.assign(
         tf.concat_v2([[smallest_new_score], new_scores], 0))
     self.last_ops = [u1, u2]
     return tf.group(u1, u2)
    def logp(self, F, Y):
        Y = tf.cast(Y, tf.int32)
        scaled_bins_left = tf.concat_v2(
            [self.bin_edges / self.sigma,
             np.array([np.inf])], 0)
        scaled_bins_right = tf.concat_v2(
            [np.array([-np.inf]), self.bin_edges / self.sigma], 0)
        selected_bins_left = tf.gather(scaled_bins_left, Y)
        selected_bins_right = tf.gather(scaled_bins_right, Y)

        return tf.log(
            probit(selected_bins_left - F / self.sigma) -
            probit(selected_bins_right - F / self.sigma) + 1e-6)
def update_tensor(V, dim2, val):  # Update tensor V, with index(:,dim2[:]) by val[:]
    print 'Shapes Recieved in Update: V, dim, val are ==> ',V.get_shape().as_list(), dim2.get_shape().as_list(), val.get_shape().as_list()
    val = tf.cast(val, V.dtype)

    def body(_, (v, d2, chg)):
        print 'Shapes Recieved in Body of Update: v, d2, chg are ==> ', v.get_shape().as_list(), d2.get_shape().as_list(), chg.get_shape().as_list()
        d2_int = tf.cast(d2, tf.int32)
        if len(chg.get_shape().as_list()) == 0:
            chg = [chg]
        else:
            chg = tf.reshape(chg, shape=[1]+chg.get_shape().as_list())
        oob = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg], axis=0), tf.range(0,len(v.get_shape().as_list())), v.get_shape().as_list())
        inb = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg, v[d2_int + 1:]], axis=0), tf.constant(0,shape=[len(v.get_shape().as_list())]), v.get_shape().as_list())
        return tf.cond(tf.less(d2_int + 1, v.get_shape().as_list()[0]), inb, oob)
Example #28
0
    def __call__(self, inputs, state, scope='attention_cell'):
        """Long short-term memory cell with attention (LSTMA)."""
        with tf.variable_scope(scope, reuse=self.reuse):
            state, attns, attn_states = state
            attn_states = tf.reshape(attn_states,
                                     [-1, self._attn_length, self._attn_size])
            input_size = self._input_size
            if input_size is None:
                input_size = inputs.get_shape().as_list()[1]
            inputs = _linear([inputs, attns],
                             input_size,
                             self.reuse,
                             trainable=self.trainable,
                             name=scope)
            lstm_output, new_state = self._cell(inputs, state)
            new_state_cat = tf.concat_v2(helper.flatten_sq(new_state), 1)
            new_attns, new_attn_states = _attention(new_state_cat,
                                                    attn_states,
                                                    True,
                                                    self.reuse,
                                                    self._attn_size,
                                                    self._attn_vec_size,
                                                    self._attn_length,
                                                    trainable=self.trainable)
            with tf.variable_scope("attn_output_projection"):
                output = _linear([lstm_output, new_attns],
                                 self._attn_size,
                                 self.reuse,
                                 trainable=self.trainable,
                                 name=scope)
            new_attn_states = tf.concat_v2(
                [new_attn_states, tf.expand_dims(output, 1)], 1)
            new_attn_states = tf.reshape(
                new_attn_states, [-1, self._attn_length * self._attn_size])
            new_state = (new_state, new_attns, new_attn_states)
            if self.layer_norm is not None:
                output = self.layer_norm(output,
                                         self.reuse,
                                         trainable=self.trainable,
                                         **self.layer_norm_args)
                new_state = self.layer_norm(new_state,
                                            self.reuse,
                                            trainable=self.trainable,
                                            **self.layer_norm_args)

            output = _collect_named_outputs(self.outputs_collections,
                                            scope + '_output', output)
            new_state = _collect_named_outputs(self.outputs_collections,
                                               scope + '_state', new_state)
            return output, new_state
Example #29
0
def UnPooling2x2ZeroFilled(x):
    # https://github.com/tensorflow/tensorflow/issues/2169
    out = tf.concat_v2([x, tf.zeros_like(x)], 3)
    out = tf.concat_v2([out, tf.zeros_like(out)], 2)

    sh = x.get_shape().as_list()
    if None not in sh[1:]:
        out_size = [-1, sh[1] * 2, sh[2] * 2, sh[3]]
        return tf.reshape(out, out_size)
    else:
        shv = tf.shape(x)
        ret = tf.reshape(out, tf.stack([-1, shv[1] * 2, shv[2] * 2, sh[3]]))
        ret.set_shape([None, None, None, sh[3]])
        return ret
def update_tensor(V, dim2, val):  # Update tensor V, with index(:,dim2[:]) by val[:]
    print 'Shapes Recieved in Update: V, dim, val are ==> ',V.get_shape().as_list(), dim2.get_shape().as_list(), val.get_shape().as_list()
    val = tf.cast(val, V.dtype)

    def body(_, (v, d2, chg)):
        print 'Shapes Recieved in Body of Update: v, d2, chg are ==> ', v.get_shape().as_list(), d2.get_shape().as_list(), chg.get_shape().as_list()
        d2_int = tf.cast(d2, tf.int32)
        if len(chg.get_shape().as_list()) == 0:
            chg = [chg]
        else:
            chg = tf.reshape(chg, shape=[1]+chg.get_shape().as_list())
        oob = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg], axis=0), tf.range(0,len(v.get_shape().as_list())), v.get_shape().as_list())
        inb = lambda : tf.slice(tf.concat_v2([v[:d2_int], chg, v[d2_int + 1:]], axis=0), tf.constant(0,shape=[len(v.get_shape().as_list())]), v.get_shape().as_list())
        return tf.cond(tf.less(d2_int + 1, v.get_shape().as_list()[0]), inb, oob)
    def _make_phi(self, F):
        """
        A helper function for making predictions. Constructs a probability
        matrix where each row output the probability of the corresponding
        label, and the rows match the entries of F.

        Note that a matrix of F values is flattened.
        """
        scaled_bins_left = tf.concat_v2(
            [self.bin_edges / self.sigma,
             np.array([np.inf])], 0)
        scaled_bins_right = tf.concat_v2(
            [np.array([-np.inf]), self.bin_edges / self.sigma], 0)
        return probit(scaled_bins_left - tf.reshape(F, (-1, 1)) / self.sigma)\
            - probit(scaled_bins_right - tf.reshape(F, (-1, 1)) / self.sigma)
 def testRandomInitUnevenPartitions(self):
     with self.test_session():
         rnd = tf.Variable(tf.random_uniform([20, 43], dtype=tf.float64))
         var_lists = [
             tf.create_partitioned_variables(rnd.get_shape(), [1, i], rnd.initialized_value()) for i in xrange(1, 10)
         ]
         tf.global_variables_initializer().run()
         rnd_val = rnd.eval()
         # Only check the slice save specs for the first 5 tf.
         save_specs = [
             # One slice
             ["20 43 0,20:0,43"],
             # Two slices
             ["20 43 0,20:0,22", "20 43 0,20:22,21"],
             # Three slices
             ["20 43 0,20:0,15", "20 43 0,20:15,14", "20 43 0,20:29,14"],
             # Four slices
             ["20 43 0,20:0,11", "20 43 0,20:11,11", "20 43 0,20:22,11", "20 43 0,20:33,10"],
             # Five slices
             ["20 43 0,20:0,9", "20 43 0,20:9,9", "20 43 0,20:18,9", "20 43 0,20:27,8", "20 43 0,20:35,8"],
         ]
         for i, vs in enumerate(var_lists):
             var_val = tf.concat_v2(vs, 1).eval()
             self.assertAllClose(rnd_val, var_val)
             self.assertEqual([tf.float64] * len(vs), [v.dtype.base_dtype for v in vs])
             if i < len(save_specs):
                 self._TestSaveSpec(vs, save_specs[i])
Example #33
0
def unzip(x, split_dim, current_length, num_splits=2, name=None):
    """Splits a tensor by unzipping along the split_dim.

  For example the following array split into 2 would be:
      [1, 2, 3, 4, 5, 6] -> [1, 3, 5], [2, 4, 6]
  and by 3:
      [1, 2, 3, 4] -> [1, 4], [2], [3]

  Args:
    x: The tensor to split.
    split_dim: The dimension to split along.
    current_length: Current length along the split_dim.
    num_splits: The number of splits.
    name: Optional name for this op.
  Returns:
    A length num_splits sequence.
  """
    with tf.name_scope(name, "unzip", [x]) as scope:
        x = tf.convert_to_tensor(x, name="x")
        # There is probably a more efficient way to do this.
        all_splits = tf.split(value=x, num_or_size_splits=current_length, axis=split_dim, name=scope)
        splits = [[] for _ in xrange(num_splits)]
        for i in xrange(current_length):
            splits[i % num_splits].append(all_splits[i])
        return [tf.concat_v2(s, split_dim) for s in splits]
Example #34
0
    def create_discriminator(discrim_inputs, discrim_targets):
        n_layers = 3
        layers = []

        # 2x [batch, height, width, in_channels] => [batch, height, width, in_channels * 2]
        input = tf.concat_v2([discrim_inputs, discrim_targets], axis=3)

        # layer_1: [batch, 256, 256, in_channels * 2] => [batch, 128, 128, ndf]
        with tf.variable_scope("layer_1"):
            convolved = conv(input, a.ndf, stride=2)
            rectified = lrelu(convolved, 0.2)
            layers.append(rectified)

        # layer_2: [batch, 128, 128, ndf] => [batch, 64, 64, ndf * 2]
        # layer_3: [batch, 64, 64, ndf * 2] => [batch, 32, 32, ndf * 4]
        # layer_4: [batch, 32, 32, ndf * 4] => [batch, 31, 31, ndf * 8]
        for i in range(n_layers):
            with tf.variable_scope("layer_%d" % (len(layers) + 1)):
                out_channels = a.ndf * min(2**(i + 1), 8)
                stride = 1 if i == n_layers - 1 else 2  # last layer here has stride 1
                convolved = conv(layers[-1], out_channels, stride=stride)
                normalized = batchnorm(convolved)
                rectified = lrelu(normalized, 0.2)
                layers.append(rectified)

        # layer_5: [batch, 31, 31, ndf * 8] => [batch, 30, 30, 1]
        with tf.variable_scope("layer_%d" % (len(layers) + 1)):
            convolved = conv(rectified, out_channels=1, stride=1)
            output = tf.sigmoid(convolved)
            layers.append(output)

        return layers[-1]
Example #35
0
def merge_outputs(tensor_list, name="MergeOutputs"):
    """ Merge Outputs.

    A layer that concatenate all outputs of a network into a single tensor.

    Input:
        List of Tensors [_shape_].

    Output:
        Concatenated Tensors [nb_tensors, _shape_].

    Arguments:
        tensor_list: list of `Tensor`. The network outputs.
        name: `str`. A name for this layer (optional).

    Returns:
        A `Tensor`.

    """
    with tf.name_scope(name) as scope:
        x = tf.concat_v2(tensor_list, 1)

    # Track output tensor.
    tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, x)

    return x
Example #36
0
def conv_cond_concat(x, y):
    """Concatenate conditioning vector on feature map axis."""
    x_shapes = x.get_shape()
    y_shapes = y.get_shape()
    return tf.concat_v2(
        [x, y * tf.ones([x_shapes[0], x_shapes[1], x_shapes[2], y_shapes[3]])],
        3)
  def testConcat(self):
    np_val = np.random.rand(3, 4, 7).astype(np.float32)
    tf_val = tf.concat_v2(
        [np_val[0:1, :, :], np_val[1:2, :, :], np_val[2:3, :, :]], 0)
    c_val = tf.contrib.util.constant_value(tf_val)
    self.assertAllClose(np_val, c_val)

    tf_val = tf.concat_v2([np_val[0, :, :], np_val[1, :, :], np_val[2, :, :]],
                          tf.placeholder(tf.int32))
    c_val = tf.contrib.util.constant_value(tf_val)
    self.assertIs(None, c_val)

    tf_val = tf.concat_v2(
        [np_val[0, :, :], tf.placeholder(tf.float32), np_val[2, :, :]], 1)
    c_val = tf.contrib.util.constant_value(tf_val)
    self.assertIs(None, c_val)
def concat(input_layer, concat_dim, other_tensors=None):
    """Concatenates input PrettyTensor with other_tensors along the specified dim.

  This adds the Pretty Tensor passed via input_layer to the front of the list of
  tensors to concat.

  Args:
    input_layer: The input layer.
    concat_dim: The dimension along which to concat.
    other_tensors: The tensors to concatenate with as an iterable or None if
      this is called on a sequence.
  Returns:
    A new PrettyTensor.
  Raises:
    ValueError: If other_tensors is None and this is not a sequence.
  """
    if input_layer.is_sequence():
        all_tensors = input_layer.sequence
        all_tensors.extend(other_tensors or [])
    else:
        all_tensors = [input_layer]
        if other_tensors is None:
            raise ValueError("Other Tensors must be supplied.")
        all_tensors.extend(other_tensors)
    # Edge cases really only apply when this is a sequence with 0 or 1 element.
    if not all_tensors:
        return prettytensor.wrap_sequence([])
    else:
        return tf.concat_v2(all_tensors, concat_dim)
Example #39
0
def one_hot_mask(labels, num_classes, scope=None):
  """Compute 1-hot encodings for masks.

  Given a label image, this computes the one hot encoding at
  each pixel.

  Args:
    labels: (batch_size, width, height, 1) tensor containing labels.
    num_classes: number of classes
    scope: optional scope name

  Returns:
    Tensor of shape (batch_size, width, height, num_classes) with
    a 1-hot encoding.
  """
  with tf.name_scope(scope, "OneHotMask", [labels]):
    height, width, depth = _shape(labels)
    assert depth == 1
    sparse_labels = tf.to_int32(tf.reshape(labels, [-1, 1]))
    sparse_size, _ = _shape(sparse_labels)
    indices = tf.reshape(tf.range(0, sparse_size, 1), [-1, 1])
    concated = tf.concat_v2([indices, sparse_labels], 1)
    dense_result = tf.sparse_to_dense(concated, [sparse_size, num_classes], 1.0,
                                      0.0)
    result = tf.reshape(dense_result, [height, width, num_classes])
    return result
Example #40
0
def horizontal_lstm(images, num_filters_out, scope=None):
  """Run an LSTM bidirectionally over all the rows of each image.

  Args:
    images: (num_images, height, width, depth) tensor
    num_filters_out: output depth
    scope: optional scope name

  Returns:
    (num_images, height, width, num_filters_out) tensor, where
    num_steps is width and new num_batches is num_image_batches * height
  """
  with tf.variable_scope(scope, "HorizontalLstm", [images]):
    batch_size, _, _, _ = _shape(images)
    sequence = images_to_sequence(images)
    with tf.variable_scope("lr"):
      hidden_sequence_lr = lstm1d.ndlstm_base(sequence, num_filters_out // 2)
    with tf.variable_scope("rl"):
      hidden_sequence_rl = (
          lstm1d.ndlstm_base(sequence,
                             num_filters_out - num_filters_out // 2,
                             reverse=1))
    output_sequence = tf.concat_v2([hidden_sequence_lr, hidden_sequence_rl], 2)
    output = sequence_to_images(output_sequence, batch_size)
    return output
def _average_gradients(tower_grads):
    """Calculate the average gradient for each shared variable across all towers.
  Note that this function provides a synchronization point across all towers.
  Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
      is over individual gradients. The inner list is over the gradient
      calculation for each tower.
  Returns:
     List of pairs of (gradient, variable) where the gradient has been averaged
     across all towers.
  """
    average_grads = []
    for grad_and_vars in zip(*tower_grads):
        # Note that each grad_and_vars looks like the following:
        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
        grads = []
        for g, _ in grad_and_vars:
            # Add 0 dimension to the gradients to represent the tower.
            expanded_g = tf.expand_dims(g, 0)

            # Append on a 'tower' dimension which we will average over below.
            grads.append(expanded_g)

        # Average over the 'tower' dimension.
        grad = tf.concat_v2(grads, 0)
        grad = tf.reduce_mean(grad, 0)

        # Keep in mind that the Variables are redundant because they are shared
        # across towers. So .. we will just return the first tower's pointer to
        # the Variable.
        v = grad_and_vars[0][1]
        grad_and_var = (grad, v)
        average_grads.append(grad_and_var)
    return average_grads
def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
    """Builds the 35x35 resnet block."""
    with tf.variable_scope(scope, 'Block35', [net], reuse=reuse):
        with tf.variable_scope('Branch_0'):
            tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1')
        with tf.variable_scope('Branch_1'):
            tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
            tower_conv1_1 = slim.conv2d(tower_conv1_0,
                                        32,
                                        3,
                                        scope='Conv2d_0b_3x3')
        with tf.variable_scope('Branch_2'):
            tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
            tower_conv2_1 = slim.conv2d(tower_conv2_0,
                                        48,
                                        3,
                                        scope='Conv2d_0b_3x3')
            tower_conv2_2 = slim.conv2d(tower_conv2_1,
                                        64,
                                        3,
                                        scope='Conv2d_0c_3x3')
        mixed = tf.concat_v2([tower_conv, tower_conv1_1, tower_conv2_2], 3)
        up = slim.conv2d(mixed,
                         net.get_shape()[3],
                         1,
                         normalizer_fn=None,
                         activation_fn=None,
                         scope='Conv2d_1x1')
        net += scale * up
        if activation_fn:
            net = activation_fn(net)
    return net
  def test_broadcast_apply_and_solve(self):
    # These cannot be done in the automated (base test class) tests since they
    # test shapes that tf.matmul cannot handle.
    # In particular, tf.matmul does not broadcast.
    with self.test_session() as sess:
      x = tf.random_normal(shape=(2, 2, 3, 4))

      # This LinearOperatorDiag will be brodacast to (2, 2, 3, 3) during solve
      # and apply with 'x' as the argument.
      diag = tf.random_uniform(shape=(2, 1, 3))
      operator = linalg.LinearOperatorDiag(diag, is_self_adjoint=True)
      self.assertAllEqual((2, 1, 3, 3), operator.shape)

      # Create a batch matrix with the broadcast shape of operator.
      diag_broadcast = tf.concat_v2((diag, diag), 1)
      mat = tf.matrix_diag(diag_broadcast)
      self.assertAllEqual((2, 2, 3, 3), mat.get_shape())  # being pedantic.

      operator_apply = operator.apply(x)
      mat_apply = tf.matmul(mat, x)
      self.assertAllEqual(operator_apply.get_shape(), mat_apply.get_shape())
      self.assertAllClose(*sess.run([operator_apply, mat_apply]))

      operator_solve = operator.solve(x)
      mat_solve = tf.matrix_solve(mat, x)
      self.assertAllEqual(operator_solve.get_shape(), mat_solve.get_shape())
      self.assertAllClose(*sess.run([operator_solve, mat_solve]))
Example #44
0
def average_gradients(tower_grads):
  """Calculate the average gradient for each shared variable across all towers.
  Note that this function provides a synchronization point across all towers.
  Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
      is over individual gradients. The inner list is over the gradient
      calculation for each tower.
  Returns:
     List of pairs of (gradient, variable) where the gradient has been averaged
     across all towers.
  """
  average_grads = []
  for grad_and_vars in zip(*tower_grads):
    # Note that each grad_and_vars looks like the following:
    #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
    grads = []
    for g, _ in grad_and_vars:
      # Add 0 dimension to the gradients to represent the tower.
      expanded_g = tf.expand_dims(g, 0)

      # Append on a 'tower' dimension which we will average over below.
      grads.append(expanded_g)

    # Average over the 'tower' dimension.
    grad = tf.concat_v2(grads, 0)
    grad = tf.reduce_mean(grad, 0)

    # Keep in mind that the Variables are redundant because they are shared
    # across towers. So .. we will just return the first tower's pointer to
    # the Variable.
    v = grad_and_vars[0][1]
    grad_and_var = (grad, v)
    average_grads.append(grad_and_var)
  return average_grads
Example #45
0
def PS(X, r, color=False):
    if color:
        Xc = tf.split(X, 3, 3)
        X = tf.concat_v2([_phase_shift(x, r) for x in Xc], 3)
    else:
        X = _phase_shift(X, r)
    return X
Example #46
0
    def testDynamicAttentionDecoderStateIsTuple(self):
      with self.test_session() as sess:
        with tf.variable_scope("root",
                               initializer=tf.constant_initializer(0.5)):
          cell = tf.contrib.rnn.BasicLSTMCell(2, state_is_tuple=True)
          cell = tf.contrib.rnn.MultiRNNCell(cells=[cell] * 2,
                                             state_is_tuple=True)
          inp = tf.constant(0.5, shape=[2, 2, 2])
          enc_outputs, enc_state = tf.contrib.rnn.static_rnn(
              cell, inp, dtype=tf.float32)
          attn_states = tf.concat_v2(
              [tf.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs],
              1)
          dec_inp = [tf.constant(0.4, shape=[2, 2])] * 3
          dec, mem = tf.contrib.legacy_seq2seq.attention_decoder(
              dec_inp, enc_state,
              attn_states, cell, output_size=4)
          sess.run([tf.global_variables_initializer()])
          res = sess.run(dec)
          self.assertEqual(3, len(res))
          self.assertEqual((2, 4), res[0].shape)

          res = sess.run([mem])
          self.assertEqual(2, len(res[0]))
          self.assertEqual((2, 2), res[0][0].c.shape)
          self.assertEqual((2, 2), res[0][0].h.shape)
          self.assertEqual((2, 2), res[0][1].c.shape)
          self.assertEqual((2, 2), res[0][1].h.shape)
Example #47
0
def linear(args, input_size, output_size, bias, bias_start=0.0, scope=None):
    """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
    Args:
        args: a 2D Tensor or a list of 2D, batch x n, Tensors.
        output_size: int, second dimension of W[i].
        bias: boolean, whether to add a bias term or not.
        bias_start: starting value to initialize the bias; 0 by default.
        scope: VariableScope for the created subgraph; defaults to "Linear".
    Returns:
        A 2D Tensor with shape [batch x output_size] equal to
        sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
    Raises:
        ValueError: if some of the arguments has unspecified or wrong shape.
    """
    if not isinstance(args, (list, tuple)):
        args = [args]

    # Calculate the total size of arguments on dimension 1.
    total_arg_size = 0
    shapes = []
    for a in args:
        try:
            shapes.append(a.get_shape().as_list())
        except Exception as e:
            shapes.append(a.shape)

    is_vector = False
    for idx, shape in enumerate(shapes):
        if len(shape) != 2:
            is_vector = True
            args[idx] = tf.reshape(args[idx], [1, -1])
    total_arg_size = input_size
    """for idx, shape in enumerate(shapes):
        if len(shape) != 2:
            is_vector = True
            args[idx] = tf.reshape(args[idx], [1, -1])
            total_arg_size += shape[0]
        else:
            total_arg_size += shape[1]"""

    # Now the computation.
    with vs.variable_scope(scope or "Linear"):
        #initializer = tf.random_normal_initializer(mean=0., stddev=1.,)
        #matrix = vs.get_variable("Matrix", [total_arg_size, output_size], initializer=initializer)
        matrix = vs.get_variable("Matrix", [total_arg_size, output_size])
        if len(args) == 1:
            res = tf.matmul(args[0], matrix)
        else:
            res = tf.matmul(tf.concat_v2(args, 1), matrix)

        if not bias:
            return res
        bias_term = vs.get_variable(
            "Bias", [output_size],
            initializer=init_ops.constant_initializer(bias_start))

    if is_vector:
        return tf.reshape(res + bias_term, [-1])
    else:
        return res + bias_term
 def testSegmentMaxGradientWithTies(self):
   inputs = tf.constant([1.0], dtype=tf.float32)
   data = tf.concat_v2([inputs, inputs], 0)
   segment_ids = tf.constant([0, 0], dtype=tf.int64)
   segment_max = tf.segment_max(data, segment_ids)
   with self.test_session():
     error = tf.test.compute_gradient_error(inputs, [1], segment_max, [1])
     self.assertLess(error, 1e-4)
    def _inference(self, docs, queries):
        """
        Computes document attentions given a document batch and query batch.
        """
        with tf.name_scope("inference"):
            # Compute document lengths / query lengths for batch
            doc_lens = length(docs)
            query_lens = length(queries)
            batch_size = tf.shape(docs)[0]

            with tf.variable_scope('encode'):
                # Encode Document / Query
                with tf.variable_scope('docs'), tf.device('/gpu:0'):
                    encoded_docs = tf.nn.dropout(self._embed(docs), self._keep_prob)
                    encoded_docs = self._bidirectional_encode(encoded_docs, doc_lens, self._encode_size)
                with tf.variable_scope('queries'), tf.device('/gpu:1'):
                    encoded_queries = tf.nn.dropout(self._embed(queries), self._keep_prob)
                    encoded_queries = self._bidirectional_encode(encoded_queries, query_lens, self._encode_size)

            with tf.variable_scope('attend') as scope:
                infer_gru = tf.nn.rnn_cell.GRUCell(self._infer_size)
                infer_state = infer_gru.zero_state(batch_size, tf.float32)
                for iter_step in range(self._num_glimpses):
                    if iter_step > 0:
                        scope.reuse_variables()

                    # Glimpse query and document
                    with tf.device('/gpu:0'):
                        q_attention, q_glimpse = self._glimpse(self._A_q, self._a_q, encoded_queries, infer_state)
                        tf.add_to_collection('query_attentions', q_attention)
                    with tf.device('/gpu:1'):
                        d_attention, d_glimpse = self._glimpse(self._A_d, self._a_d, encoded_docs, tf.concat_v2([infer_state, q_glimpse], 1))
                        tf.add_to_collection('doc_attentions', d_attention)
                    # Search Gates

                    gate_concat = tf.concat_v2([infer_state, q_glimpse, d_glimpse, q_glimpse * d_glimpse], 1)

                    r_d = tf.sigmoid(tf.matmul(gate_concat, self._g_d))
                    r_d = tf.nn.dropout(r_d, self._keep_prob)
                    r_q = tf.sigmoid(tf.matmul(gate_concat, self._g_q))
                    r_q = tf.nn.dropout(r_q, self._keep_prob)

                    combined_gated_glimpse = tf.concat_v2([r_q * q_glimpse, r_d * d_glimpse], 1)
                    _, infer_state = infer_gru(combined_gated_glimpse, infer_state)

            return tf.to_float(tf.sign(tf.abs(docs))) * d_attention
Example #50
0
 def LSTMCell(cls, x, mprev, cprev, weights):
   xm = tf.concat_v2([x, mprev], 1)
   i_i, i_g, f_g, o_g = tf.split(
       value=tf.matmul(xm, weights), num_or_size_splits=4, axis=1)
   new_c = tf.sigmoid(f_g) * cprev + tf.sigmoid(i_g) * tf.tanh(i_i)
   new_c = tf.clip_by_value(new_c, -50.0, 50.0)
   new_m = tf.sigmoid(o_g) * tf.tanh(new_c)
   return new_m, new_c
Example #51
0
    def __call__(self, input_tuple, state, scope=None):
        with vs.variable_scope(scope or type(self).__name__):
            self._init_parameters()
        
            _input, target = input_tuple

            assert _input.get_shape()[2] == 1, "Need input depth == 1" 
            _input = tf.squeeze(_input, 2)
            
            u, s, r, inner_spikes, spike_history, dW, dF, dR = state
            
            _input_filtered = tf.matmul(_input, self.F)

            s = (1.0 - tau_syn) * s + tf.concat_v2([_input_filtered, inner_spikes], 1)
            u = (1.0 - tau_mem) * u + mo.matmul(s, self.W)
            r = (1.0 - tau_refr) * r
            

            act_raw = self._activation(u)
            act = act_raw * tf.exp(-r)
            
            spikes = tf.where(
                act > tf.random_uniform([batch_size, self._num_units]),
                tf.ones([batch_size, self._num_units]),
                tf.zeros([batch_size, self._num_units])
            )   

            r += spikes * amp_refr
            
            spike_history = cat_hist(spike_history, spikes, 1)

            act_grad = tf.gradients([act], [u])[0]
            
            neuron_derivative = (act_grad/act_raw) * (spikes - act) 

            spike_history_reshaped = tf.reshape(tf.transpose(spike_history, [0, 2, 1]), [batch_size * self._num_units, L])
            
            output_point = tf.matmul(spike_history_reshaped, self.R)
            output_point = tf.reshape(output_point, [batch_size, self._num_units])
            output_point = tf.reduce_sum(output_point, 1)
            output_point = tf.expand_dims(output_point, 1)
            
            loss = tf.reduce_sum(tf.square(output_point - target), 1)/2.0
            loss = tf.expand_dims(loss, 1)
            
            dR += - lrate * tf.gradients([loss], [self.R])[0]
            
            sp = - tf.gradients([loss], [spike_history])[0]
            sp = tf.reduce_sum(sp, [1,2])
            sp = tf.expand_dims(sp, 1)
            
            dW += lrate * outer(
                tf.reduce_sum(s, 0),
                tf.reduce_sum(neuron_derivative * sp, 0)
            )
            

            return GLMOutputTuple(spikes, act, output_point, loss), GLMStateTuple(u, s, r, spikes, spike_history, dW, dF, dR)
 def testDegenerate(self):
     with self.test_session():
         rnd = tf.Variable(tf.random_uniform([10, 43]))
         vs = tf.create_partitioned_variables(rnd.get_shape(), [1, 1], rnd.initialized_value())
         tf.global_variables_initializer().run()
         val = tf.concat_v2(vs, 0).eval()
         rnd = rnd.eval()
         self.assertAllClose(rnd, val)
         self._TestSaveSpec(vs, ["10 43 0,10:0,43"])
Example #53
0
 def remove(self, ids):
     """Remove the ids (and their associated scores) from the TopN."""
     with tf.control_dependencies(self.last_ops):
         scatter_op = tf.scatter_update(self.id_to_score, ids, tf.ones_like(ids, dtype=tf.float32) * tf.float32.min)
         # We assume that removed ids are almost always in the shortlist,
         # so it makes no sense to hide the Op behind a tf.cond
         shortlist_ids_to_remove, new_length = tensor_forest_ops.top_n_remove(self.sl_ids, ids)
         u1 = tf.scatter_update(
             self.sl_ids,
             tf.concat_v2([[0], shortlist_ids_to_remove], 0),
             tf.concat_v2([new_length, tf.ones_like(shortlist_ids_to_remove) * -1], 0),
         )
         u2 = tf.scatter_update(
             self.sl_scores,
             shortlist_ids_to_remove,
             tf.float32.min * tf.ones_like(shortlist_ids_to_remove, dtype=tf.float32),
         )
         self.last_ops = [scatter_op, u1, u2]
Example #54
0
  def test(self):
    condition = core.LabeledTensor(tf.range(5) < 3, ['x'])
    x = core.LabeledTensor(tf.ones(5), ['x'])
    y = core.LabeledTensor(tf.zeros(5), ['x'])
    where_lt = ops.where(condition, x, y)

    golden_lt = core.LabeledTensor(
        tf.concat_v2([tf.ones(3), tf.zeros(2)], 0), ['x'])
    self.assertLabeledTensorsEqual(where_lt, golden_lt)
Example #55
0
    def __call__(self, x, h, scope=None):
        x_t = fun(x, nout = x_transformed, act = tf.nn.tanh, name = "x_transformed", weight_factor = weight_factor, layers_num = layers_num)

        # prior, depends only on state

        prior_t = fun(h, nout = prior_interm, act = tf.nn.tanh, name = "prior", weight_factor = weight_factor, layers_num = layers_num)
        prior_mu_t = fun(prior_t, nout = z_dim, act = tf.identity, name = "prior_mu", weight_factor = weight_factor)
        prior_sigma_t = fun(prior_t, nout = z_dim, act = tf.nn.softplus, name = "prior_sigma", weight_factor = weight_factor)

        # phi
        phi_t = fun(x_t, h, nout = phi_interm, act = tf.nn.tanh, name = "phi", weight_factor = weight_factor, layers_num = layers_num)
        phi_mu_t = fun(phi_t, nout = z_dim, act = tf.identity, name = "phi_mu", weight_factor = weight_factor)
        phi_sigma_t = fun(phi_t, nout = z_dim, act = tf.nn.softplus, name = "phi_sigma", weight_factor = weight_factor)

        # z generating
        epsilon = tf.random_normal(tf.shape(z_dim), name='epsilon')

        if not self._generator:
            z = phi_mu_t + phi_sigma_t * epsilon
        else:
            z = prior_mu_t + prior_sigma_t * epsilon

        z_t = fun(z, nout = z_interm, act = tf.nn.tanh, name = "z_transformed", weight_factor = weight_factor, layers_num = layers_num)

        output_t = fun(z_t, h, nout = out_interm, act = tf.nn.tanh, name = "out_transform", weight_factor = weight_factor, layers_num = layers_num)
        output_mu = fun(output_t, nout = 1, act =tf.identity, name = "out_mu", weight_factor = weight_factor)
        # output_sigma = fun(out_t, nout = 1, act = tf.nn.softplus, name = "out_sigma", weight_factor = weight_factor)

        if not self._generator:
            x_c = tf.concat_v2([x_t, z_t], 1)
        else:
            x_t_gen = fun(output_mu, nout = x_transformed, act = tf.nn.tanh, name = "x_transformed", weight_factor = weight_factor, layers_num = layers_num)
            x_c = tf.concat_v2([x_t_gen, z_t], 1)

        _, new_h = self._base_cell(x_c, h)

        return VAEOutputTuple(
            prior_mu_t,
            prior_sigma_t,
            phi_mu_t,
            phi_sigma_t,
            z,
            z_t,
            output_mu), new_h
 def testVecConstantInit(self):
     with self.test_session():
         rnd_par = tf.constant([1, 2, 3, 4])
         vs = tf.create_partitioned_variables([4], [4], rnd_par)
         tf.global_variables_initializer().run()
         val = tf.concat_v2(vs, 0).eval()
         rnd = rnd_par.eval()
         self.assertAllClose(rnd, val)
         self.assertEqual([tf.int32] * 4, [v.dtype.base_dtype for v in vs])
         self._TestSaveSpec(vs, ["4 0,1", "4 1,1", "4 2,1", "4 3,1"])
 def testConstantInit(self):
     with self.test_session():
         rnd_par = tf.constant([[1, 2, 3, 4], [5, 6, 7, 8]])
         vs = tf.create_partitioned_variables([2, 4], [1, 2], rnd_par)
         tf.global_variables_initializer().run()
         val = tf.concat_v2(vs, 1).eval()
         rnd = rnd_par.eval()
         self.assertAllClose(rnd, val)
         self.assertEqual([tf.int32] * 2, [v.dtype.base_dtype for v in vs])
         self._TestSaveSpec(vs, ["2 4 0,2:0,2", "2 4 0,2:2,2"])
Example #58
0
 def testConcatLargeTensors(self):
   # CPU-only test, because it fails on GPUs with <= 4GB memory.
   with tf.device("/cpu:0"):
     a = tf.ones([2**31 + 6], dtype=tf.int8)
     b = tf.zeros([1024], dtype=tf.int8)
     onezeros = tf.concat_v2([a, b], 0)
   with self.test_session(use_gpu=False):
     # TODO(dga):  Add more depth to this test to validate correctness,
     # not just non-crashingness, once other large tensor fixes have gone in.
     _ = onezeros.eval()