예제 #1
0
        def forward_pass(s):

            s = tf.layers.batch_normalization(s, name='bn')
            fc1 = tf.layers.Dense(units=256, activation=tf.nn.relu,
                                  name='fc1')(s)

            # fc_V
            fc_V = tf.layers.Dense(64, activation=tf.nn.relu,
                                   name='fc_V_1')(fc1)
            self.fc_V = tf.layers.Dense(1, activation=None,
                                        name='fc_V_2')(fc_V)

            # fc_A
            fc_A = tf.layers.Dense(32, activation=tf.nn.relu,
                                   name='fc_A_1')(fc1)
            self.fc_A = tf.layers.Dense(self.n_actions,
                                        activation=None,
                                        name='fc_A_2')(fc_A)

            with tf.variable_scope('q_predict'):
                mean_A = tf.reduce_mean(self.fc_A, axis=1, keep_dims=True)
                q_predict = tf.add(self.fc_V, tf.subtract(self.fc_A, mean_A))

            return q_predict
예제 #2
0
def fcn4rnn(nn_input, dims, weights, biases, dim_hidden, flag):
    cur_top = nn_input
    n_layers = len(dim_hidden) - 1
    for layer_step in range(n_layers):
        top = dict()
        for i in range(dim_hidden[layer_step+1]):
            top[i] = weights[layer_step][0, i]*cur_top[:, 0:dims]
            for j in range(1, dim_hidden[layer_step]):
                loc = cur_top[:, dims*j:dims*(j+1)]
                #if flag == 'outer':
                #    print(tf.shape(top[i]))
                #    print(tf.shape(loc))
                #    print(weights[layer_step][j, i])
                top[i] = tf.add(top[i], weights[layer_step][j,i]*loc)
            top[i] = top[i] + biases[layer_step][i]
        cur_top = top[0]
        for i in range(1, dim_hidden[layer_step+1]):
            cur_top = tf.concat([cur_top, top[i]], axis=1)
        if flag == 'inner':
            cur_top = tf.nn.tanh(cur_top)
        elif flag == 'outer':
            if layer_step != n_layers - 1:
                cur_top = tf.nn.relu(cur_top)
    return cur_top
예제 #3
0
            def dp_body(k, logZ, lastZ):
                '''
        case j < N-k + 1:
          logZ[j,k] = log_sum( log(1-pi(j)) + logZ[j+1,k], logp(j) + logZ[j+1,k-1])
        case j = N-k + 1
          logZ[j,k] = accum_logp[j]
        case j > N-k + 1
          logZ[j,k] = 0
        '''

                # shift lastZ one step
                shifted_lastZ = tf.roll(lastZ[:, :-1], shift=1,
                                        axis=1)  #logZ[j+1,k-1]
                log_yes = logp + shifted_lastZ  # b x N
                logZ_j = tf.TensorArray(tf.float32, size=seq_len + 1)
                init_value = accum_logp[:, seq_len - k]
                logZ_j = logZ_j.write(seq_len - k, init_value)
                _, logZ_j, logb, loga = tf.while_loop(
                    accum_cond, accum_body,
                    [seq_len - k - 1, logZ_j, log_yes, init_value])
                logZ_j = logZ_j.stack()  # N x b
                logZ_j = tf.transpose(logZ_j, [1, 0])  # b x N
                logZ = logZ.write(k, logZ_j)
                return [tf.add(k, 1), logZ, logZ_j]
예제 #4
0
파일: gan.py 프로젝트: garthtrickett/tframe
    def _define_losses(self, loss, alpha):
        if callable(loss):
            self._loss_G, self._loss_D = loss(self)
            return
        elif not isinstance(loss, six.string_types):
            raise TypeError('loss must be callable or a string')

        loss = loss.lower()
        loss_Dr_raw, loss_Df_raw, loss_G_raw = None, None, None
        if loss == pedia.default:
            loss_Dr_raw = -tf.log(self._Dr, name='loss_D_real_raw')
            loss_Df_raw = -tf.log(1. - self._Df, name='loss_D_fake_raw')
            loss_G_raw = -tf.log(self._Df, name='loss_G_raw')
        elif loss == pedia.cross_entropy:
            loss_Dr_raw = tf.nn.sigmoid_cross_entropy_with_logits(
                logits=self._logits_Dr,
                labels=tf.ones_like(self._logits_Dr) * alpha)
            loss_Df_raw = tf.nn.sigmoid_cross_entropy_with_logits(
                logits=self._logits_Df, labels=tf.zeros_like(self._logits_Df))
            loss_G_raw = tf.nn.sigmoid_cross_entropy_with_logits(
                logits=self._logits_Df, labels=tf.ones_like(self._logits_Df))
        else:
            raise ValueError('Can not resolve "{}"'.format(loss))

        reg_loss_D = self.D.regularization_loss
        reg_loss_G = self.G.regularization_loss
        with tf.name_scope('D_losses'):
            self._loss_Dr = tf.reduce_mean(loss_Dr_raw, name='loss_D_real')
            self._loss_Df = tf.reduce_mean(loss_Df_raw, name='loss_D_fake')
            self._loss_D = tf.add(self._loss_Dr, self._loss_Df, name='loss_D')
            self._loss_D = (self._loss_D if reg_loss_D is None else
                            self._loss_D + reg_loss_D)
        with tf.name_scope('G_loss'):
            self._loss_G = tf.reduce_mean(loss_G_raw, name='loss_G')
            self._loss_G = (self._loss_G if reg_loss_G is None else
                            self._loss_G + reg_loss_G)
    def conv_layer(self, idx, inputs, filters, size, stride):
        channels = inputs.get_shape()[3]
        weight = tf.Variable(
            tf.truncated_normal(
                [size, size, int(channels), filters], stddev=0.1))
        biases = tf.Variable(tf.constant(0.1, shape=[filters]))

        pad_size = size // 2
        pad_mat = np.array([[0, 0], [pad_size, pad_size], [pad_size, pad_size],
                            [0, 0]])
        inputs_pad = tf.pad(inputs, pad_mat)

        conv = tf.nn.conv2d(inputs_pad,
                            weight,
                            strides=[1, stride, stride, 1],
                            padding='VALID',
                            name=str(idx) + '_conv')
        conv_biased = tf.add(conv, biases, name=str(idx) + '_conv_biased')
        print(
            'Layer  %d : Type = Conv, Size = %d * %d, Stride = %d, Filters = %d, Input channels = %d'
            % (idx, size, size, stride, filters, int(channels)))
        return tf.maximum(self.alpha * conv_biased,
                          conv_biased,
                          name=str(idx) + '_leaky_relu')
예제 #6
0
    def call(self, inputs, training=True, drop_connect_rate=None):
        """Implementation of call().

    Args:
      inputs: the inputs tensor.
      training: boolean, whether the model is constructed for training.
      drop_connect_rate: float, between 0 to 1, drop connect rate.

    Returns:
      A output tensor.
    """
        logging.info('Block input: %s shape: %s', inputs.name, inputs.shape)
        if self._block_args.expand_ratio != 1:
            x = self._relu_fn(
                self._bn0(self._expand_conv(inputs), training=training))
        else:
            x = inputs
        logging.info('Expand: %s shape: %s', x.name, x.shape)

        self.endpoints = {'expansion_output': x}

        x = self._bn1(self._project_conv(x), training=training)
        # Add identity so that quantization-aware training can insert quantization
        # ops correctly.
        x = tf.identity(x)

        if self._block_args.id_skip:
            if all(
                    s == 1 for s in self._block_args.strides
            ) and self._block_args.input_filters == self._block_args.output_filters:
                # only apply drop_connect if skip presents.
                if drop_connect_rate:
                    x = utils.drop_connect(x, training, drop_connect_rate)
                x = tf.add(x, inputs)
        logging.info('Project: %s shape: %s', x.name, x.shape)
        return x
예제 #7
0
  def loss(self, policy_logits, action_values):
    """Constructs a TF graph that computes the RPG loss for batches.

    Args:
      policy_logits: `B x A` tensor corresponding to policy logits.
      action_values: `B x A` tensor corresponding to Q-values.

    Returns:
      loss: A 0-D `float` tensor corresponding the loss.
    """
    _assert_rank_and_shape_compatibility([policy_logits, action_values], 2)
    regrets = compute_regrets(policy_logits, action_values)
    _assert_rank_and_shape_compatibility([regrets], 1)
    total_regret = tf.reduce_mean(regrets, axis=0)

    total_loss = total_regret
    if self._entropy_cost:
      policy_entropy = tf.reduce_mean(compute_entropy(policy_logits))
      entropy_loss = tf.multiply(
          float(self._entropy_cost), policy_entropy, name="entropy_loss")
      total_loss = tf.add(
          total_loss, entropy_loss, name="total_loss_with_entropy")

    return total_loss
예제 #8
0
def res_block_2d(input,
                 out_channels=64,
                 scope='res_block',
                 kernel=[3, 3],
                 prob=0.5,
                 stride=[1, 1],
                 weight_dict=None):
    with tf.variable_scope(scope):
        net = tf.nn.relu(
            conv2d(input,
                   out_channels,
                   kernel_size=kernel,
                   stride=stride,
                   pad="SAME",
                   scope="con1_3X3",
                   weight_initializer=get_weight(scope + 'con1_3X3_weights',
                                                 weight_dict),
                   bias_initializer=get_weight(scope + 'con1_3X3_biases',
                                               weight_dict),
                   weight_initializer_type=tf.contrib.layers.
                   xavier_initializer()))
        # net = tf.nn.dropout(net, keep_prob(prob, is_training))
        net = conv2d(
            net,
            out_channels,
            kernel_size=kernel,
            stride=stride,
            pad="SAME",
            scope="conv2_3x3",
            weight_initializer=get_weight(scope + 'conv2_3x3_weights',
                                          weight_dict),
            bias_initializer=get_weight(scope + 'conv2_3x3_biases',
                                        weight_dict),
            weight_initializer_type=tf.contrib.layers.xavier_initializer())
        # net = tf.nn.dropout(net, keep_prob(prob, is_training))
    return tf.add(tf.cast(net, tf.float32), tf.cast(input, tf.float32))
예제 #9
0
    def _build_model(self):
        self.graph_built = True
        tf.set_random_seed(self.seed)
        self.labels = tf.placeholder(tf.float32, shape=[None])
        self.is_training = tf.placeholder_with_default(False, shape=[])
        self.linear_embed, self.pairwise_embed = [], []

        self._build_user_item()
        if self.sparse:
            self._build_sparse()
        if self.dense:
            self._build_dense()

        linear_embed = tf.concat(self.linear_embed, axis=1)
        pairwise_embed = tf.concat(self.pairwise_embed, axis=1)

        #    linear_term = tf.reduce_sum(linear_embed, axis=1,
        #                                keepdims=True)

        # B * 1
        linear_term = tf.layers.dense(linear_embed, units=1, activation=None)
        # B * K
        pairwise_term = 0.5 * tf.subtract(
            tf.square(tf.reduce_sum(pairwise_embed, axis=1)),
            tf.reduce_sum(tf.square(pairwise_embed), axis=1))

        #    For original FM, just add K dim together:
        #    pairwise_term = 0.5 * tf.reduce_sum(pairwise_term, axis=1)
        if self.use_bn:
            pairwise_term = tf.layers.batch_normalization(
                pairwise_term, training=self.is_training)
        pairwise_term = tf.layers.dense(inputs=pairwise_term,
                                        units=1,
                                        activation=tf.nn.elu)
        self.output = tf.squeeze(tf.add(linear_term, pairwise_term))
        count_params()
예제 #10
0
    def call(self, inputs, training=True):
        """Implementation of MnasBlock call().

    Args:
      inputs: the inputs tensor.
      training: boolean, whether the model is constructed for training.

    Returns:
      A output tensor.
    """
        tf.logging.info('Block input: %s shape: %s' %
                        (inputs.name, inputs.shape))
        if self._block_args.expand_ratio != 1:
            x = tf.nn.relu(
                self._bn0(self._expand_conv(inputs), training=training))
        else:
            x = inputs
        tf.logging.info('Expand: %s shape: %s' % (x.name, x.shape))

        x = tf.nn.relu(self._bn1(self._depthwise_conv(x), training=training))
        tf.logging.info('DWConv: %s shape: %s' % (x.name, x.shape))

        if self.has_se:
            with tf.variable_scope('se'):
                x = self._call_se(x)

        self.endpoints = {'expansion_output': x}

        x = self._bn2(self._project_conv(x), training=training)
        if self._block_args.id_skip:
            if all(
                    s == 1 for s in self._block_args.strides
            ) and self._block_args.input_filters == self._block_args.output_filters:
                x = tf.add(x, inputs)
        tf.logging.info('Project: %s shape: %s' % (x.name, x.shape))
        return tf.identity(x)
    def predict(self, X):
        assert self.dist is not None
        nb_classes, nb_features = map(int, self.dist.scale.shape)

        # Conditional probabilities log P(x|c) with shape
        # (nb_samples, nb_classes)
        cond_probs = tf.reduce_sum(self.dist.log_prob(
            tf.reshape(tf.tile(X, [1, nb_classes]),
                       [-1, nb_classes, nb_features])),
                                   axis=2)

        # uniform priors
        priors = np.log(np.array([1. / nb_classes] * nb_classes))

        # posterior log probability, log P(c) + log P(x|c)
        joint_likelihood = tf.add(priors, cond_probs)

        # normalize to get (log)-probabilities
        norm_factor = tf.reduce_logsumexp(joint_likelihood,
                                          axis=1,
                                          keep_dims=True)
        log_prob = joint_likelihood - norm_factor
        # exp to get the actual probabilities
        return tf.exp(log_prob)
예제 #12
0
def MultiscaleBlock_2(inputs, filters_1, filters_2, filters_3, p, d):
    net_1 = tf.nn.relu(slim.batch_norm(inputs, fused=True))
    net_1 = slim.conv2d(net_1,
                        filters_1, [1, 1],
                        activation_fn=None,
                        normalizer_fn=None)

    scale_1 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
    scale_1 = slim.conv2d(scale_1,
                          filters_3 // 2, [3, 3],
                          rate=p,
                          activation_fn=None,
                          normalizer_fn=None)
    scale_2 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
    scale_2 = slim.conv2d(scale_2,
                          filters_3 // 2, [3, 3],
                          rate=d,
                          activation_fn=None,
                          normalizer_fn=None)
    net_1 = tf.concat((scale_1, scale_2), axis=-1)

    net_1 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
    net_1 = slim.conv2d(net_1,
                        filters_2, [1, 1],
                        activation_fn=None,
                        normalizer_fn=None)

    net_2 = tf.nn.relu(slim.batch_norm(inputs, fused=True))
    net_2 = slim.conv2d(net_2,
                        filters_2, [1, 1],
                        activation_fn=None,
                        normalizer_fn=None)

    net = tf.add(net_1, net_2)

    return net
예제 #13
0
def zero_hidden_model(X_train, y_train, X_test, y_test, iter_num=2000):
    tf.reset_default_graph()
    n_feature = X_train.shape[1]
    X = tf.placeholder(tf.float32, shape=(None, n_feature))
    Y = tf.placeholder(tf.float32, shape=(None))
    w1 = tf.get_variable(name='w1',
                         shape=(n_feature, 1),
                         dtype=tf.float32,
                         initializer=tf.keras.initializers.glorot_uniform())
    b1 = tf.get_variable(name='b1',
                         shape=(1, 1),
                         dtype=tf.float32,
                         initializer=tf.zeros_initializer())
    z = tf.reshape(tf.add(tf.matmul(X, w1), b1), [-1])
    loss = tf.reduce_mean(
        tf.nn.sigmoid_cross_entropy_with_logits(labels=Y, logits=z))
    opt = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
    init = tf.global_variables_initializer()
    train_acc = 0.
    test_acc = 0.
    iter_list = []
    cost_list = []
    with tf.Session() as sess:
        sess.run(init)
        for iter_i in range(iter_num):
            _, cost = sess.run([opt, loss], feed_dict={X: X_train, Y: y_train})
            if iter_i % 10 == 0:
                iter_list.append(iter_i)
                cost_list.append(cost)
        train_predict = np.array(sess.run(z, feed_dict={X: X_train}) > 0,
                                 dtype=int)
        test_predict = np.array(sess.run(z, feed_dict={X: X_test}) > 0,
                                dtype=int)
        train_acc = np.sum(train_predict == y_train) / len(train_predict)
        test_acc = np.sum(test_predict == y_test) / len(test_predict)
    return (iter_list, cost_list), (train_acc, test_acc)
예제 #14
0
def create_dual_approx(num_layers, batch_size, action_max, W_T_list, b_T_list,
                       action_tensor_center, return_full_info=False):

  #layers_n: number of hidden units each layer
  #W_T_list, b_T_list: multiplicatie and bias weights for each layer
  #action_tensor_center: raw input, y: one-hot encoding of labels

  # List of bounds (l_i,u_i) for i = 2,...,K-1
  l_list = [tf.zeros_like(action_tensor_center)]
  u_list = [tf.zeros_like(action_tensor_center)]

  # List of transition matrices D_i for i = 2,...,K-1
  D_list = [tf.zeros_like(action_tensor_center)]

  # Indicators of spanning ReLu neurons for i = 2,...,K-1
  I_list = [tf.zeros_like(action_tensor_center)]

  # Indicators of active ReLu neurons for i = 2,...,K-1
  Ip_list = [tf.zeros_like(action_tensor_center)]

  # Final list of duals nu_i for i = 2,...,K-1
  Nu_list = [tf.zeros([batch_size, W_T_list[0].get_shape().as_list()[1], 1])]

  # Initialize Nu_K
  Nu_K = -tf.expand_dims(-tf.eye(1), axis=-1)

  # Final list of b_i'*nu_{i+1} for i = 1,...,K-1
  gamma_list = [b_T_list[0]]

  # Pre-compute bounds for layer 2
  # Initialize Nu_hat_1
  Nu_hat_1 = tf.tile(tf.expand_dims(W_T_list[0], axis=0), [batch_size, 1, 1])

  # Initialize bounds
  l_2 = tf.matmul(action_tensor_center,
                  W_T_list[0]) + gamma_list[0] - action_max * tf.norm(
                      Nu_hat_1, 1, axis=1, keepdims=False)
  u_2 = tf.matmul(action_tensor_center,
                  W_T_list[0]) + gamma_list[0] + action_max * tf.norm(
                      Nu_hat_1, 1, axis=1, keepdims=False)

  # Add to list (store in vector format)
  l_list.append(l_2)
  u_list.append(u_2)

  # Recursion

  for i in range(2, num_layers):
    # form Ip, I
    Ip_i, I_i = get_I(l_list[i - 1], u_list[i - 1])
    I_list.append(I_i)
    Ip_list.append(Ip_i)

    # form D
    D_i = get_D(l_list[i - 1], u_list[i - 1], Ip_i, I_i)
    D_list.append(D_i)

    # initialize nu_i
    Nu_list.append(tf.einsum('ij,jk->ijk', D_i, W_T_list[i - 1]))

    # initialize gamma_i
    gamma_list.append(b_T_list[i - 1])

    # if final iteration, update with Nu_K
    if i == num_layers - 1:
      Nu_K = tf.tile(Nu_K, [Nu_list[i - 1].get_shape().as_list()[0], 1, 1])
      Nu_list[i - 1] = tf.einsum('ijk,ikm->ijm', Nu_list[i - 1], Nu_K)
      gamma_list[i - 1] = tf.einsum('ij,ijm->im', gamma_list[i - 1], Nu_K)

    # initialize next layer bounds
    l_ip1 = tf.einsum('ij,ijm->im', l_list[i - 1] * I_list[i - 1],
                      tf.nn.relu(-Nu_list[i - 1]))
    u_ip1 = -tf.einsum('ij,ijm->im', l_list[i - 1] * I_list[i - 1],
                       tf.nn.relu(Nu_list[i - 1]))

    # update nu for layers i-1,...,2
    for j in range(i - 1, 1, -1):
      Nu_hat_j = tf.einsum('jk,ikm->ijm', W_T_list[j - 1], Nu_list[j])

      Nu_list[j - 1] = tf.einsum('ij,ijk->ijk', D_list[j - 1], Nu_hat_j)

      l_ip1 = tf.add(
          l_ip1,
          tf.einsum('ij,ijm->im', l_list[j - 1] * I_list[j - 1],
                    tf.nn.relu(-Nu_list[j - 1])))
      u_ip1 = tf.subtract(
          u_ip1,
          tf.einsum('ij,ijm->im', l_list[j - 1] * I_list[j - 1],
                    tf.nn.relu(Nu_list[j - 1])))

    # update nu_hat_1
    Nu_hat_1 = tf.einsum('jk,ikm->ijm', W_T_list[0], Nu_list[1])

    # start sum
    psi = tf.einsum('ij,ijm->im', action_tensor_center,
                    Nu_hat_1) + gamma_list[i - 1]

    # update gamma for layers 1,...,i-1
    for j in range(1, i):
      gamma_list[j - 1] = tf.einsum('ij,ijm->im', b_T_list[j - 1], Nu_list[j])

      psi = tf.add(psi, gamma_list[j - 1])

    Nu_hat_1_norm = tf.norm(Nu_hat_1, 1, axis=1, keepdims=False)

    if i < num_layers - 1:
      # finalize bounds
      l_ip1 = tf.add(l_ip1, psi - action_max * Nu_hat_1_norm)
      u_ip1 = tf.add(u_ip1, psi + action_max * Nu_hat_1_norm)

      # add to list
      l_list.append(l_ip1)
      u_list.append(u_ip1)

    else:
      # compute J_tilde
      J_tilde = -psi - action_max * Nu_hat_1_norm - u_ip1

  if return_full_info:
    return (-J_tilde, l_list, u_list, D_list, Nu_list, gamma_list, psi, l_ip1,
            u_ip1, Nu_hat_1)
  else:
    return -J_tilde
예제 #15
0
def linear_regression_categorical():
    raw_train_dataset = library.data_processing(train_data_path)
    dummies = pd.get_dummies(pd.DataFrame(raw_train_dataset[['wd']]))
    X_d = dummies.to_numpy()
    Y_d = pd.DataFrame(raw_train_dataset[['PM2.5']]).to_numpy()

    X = tf.placeholder(tf.float32, name='x')
    Y = tf.placeholder(tf.float32, name='y')

    w = tf.Variable(np.random.normal(), name='weight')
    b = tf.Variable(np.random.normal(), name='bias')

    y_pred = tf.add(tf.multiply(X, w), b)

    loss = tf.reduce_sum(tf.square(y_pred - Y)) / (2 * X_d.shape[0])
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

    init = tf.global_variables_initializer()

    # Launch the graph
    with tf.Session() as sess:
        sess.run(init)
        # Fit all training data
        for epoch in range(training_epochs):
            for (x, y) in zip(X_d, Y_d):
                x = x.reshape(1, X_d.shape[1])
                sess.run(optimizer, feed_dict={X: x, Y: y})

            # Display logs per epoch step
            if epoch % display_step == 0:
                print(
                    "Epoch:", '%04d' % (epoch + 1), "cost=",
                    "{:.9f}".format(sess.run(loss, feed_dict={
                        X: X_d,
                        Y: Y_d
                    })), "W=", sess.run(w), "b=", sess.run(b))

                fig = plt.figure(figsize=(10, 10), dpi=100)
                ax = raw_train_dataset.plot.scatter(x='wd', y='PM2.5')
                ax.set_ylim(0, 1)

                ax.plot(X_d,
                        sess.run(w) * X_d + sess.run(b),
                        label='Fitted line')
                ax.legend()
                plt.show()
                plt.close(fig)
        print("Optimization Finished!")
        training_cost = sess.run(loss, feed_dict={X: X_d, Y: Y_d})
        t_w = sess.run(w)
        t_b = sess.run(b)
        print("Training cost=", training_cost, "W=", t_w, "b=", t_b, '\n')

        raw_test_dataset = library.data_processing(test_data_path)
        X_test_d = pd.DataFrame(raw_test_dataset[['wd']]).to_numpy()
        dummies = pd.get_dummies(pd.DataFrame(raw_test_dataset[['wd']]))
        X_d = dummies.to_numpy()
        Y_test_d = pd.DataFrame(raw_test_dataset[['PM2.5']]).to_numpy()
        print("Testing... (L2 loss Comparison)")
        testing_cost = sess.run(tf.reduce_sum(tf.pow(y_pred - Y, 2)) /
                                (2 * X_test_d.shape[0]),
                                feed_dict={
                                    X: X_d,
                                    Y: Y_test_d
                                })
        print("Testing cost=", testing_cost)
        print("Absolute l2 loss difference:",
              abs(training_cost - testing_cost))
예제 #16
0
def simple_linear_regression():
    raw_train_dataset = library.data_processing(train_data_path)
    X_d = pd.DataFrame(raw_train_dataset[[
        'TEMP'
    ]]).to_numpy()  # Change the variables here to train using different values
    Y_d = pd.DataFrame(raw_train_dataset[['PM2.5']]).to_numpy()

    X = tf.placeholder(tf.float32, [X_d.shape[0], X_d.shape[1]], name='x')
    Y = tf.placeholder(tf.float32, name='y')

    w = tf.Variable(np.random.normal(), [None, X_d.shape[1]], name='weight')
    b = tf.Variable(np.random.normal(), name='bias')

    y_pred = tf.add(tf.multiply(X, w), b)

    loss = tf.reduce_sum(tf.square(y_pred - Y)) / (2 * X_d.shape[0])
    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

    init = tf.global_variables_initializer()

    # Launch the graph
    with tf.Session() as sess:
        sess.run(init)
        count = 0
        # Fit all training data
        for epoch in range(training_epochs):
            for (x, y) in zip(X_d, Y_d):
                sess.run(optimizer, feed_dict={X: x, Y: y})

            # Display logs per epoch step
            if epoch % display_step == 0:
                print(
                    "Epoch:", '%04d' % (epoch + 1), "cost=",
                    "{:.9f}".format(sess.run(loss, feed_dict={
                        X: X_d,
                        Y: Y_d
                    })), "W=", sess.run(w), "b=", sess.run(b))

                fig = plt.figure(figsize=(10, 10), dpi=100)
                ax = fig.add_subplot(111)
                ax.set_ylim(0, 1)
                ax.plot(X_d, Y_d, 'ro', label='Original data')

                ax.plot(X_d,
                        sess.run(w) * X_d + sess.run(b),
                        label='Fitted line')
                ax.legend()
                plt.show()
                fig.savefig('plot_{:05d}.png'.format(count),
                            bbox_inches='tight',
                            dpi=100)
                count = count + 1
                plt.close(fig)
        print("Optimization Finished!")
        training_cost = sess.run(loss, feed_dict={X: X_d, Y: Y_d})
        t_w = sess.run(w)
        t_b = sess.run(b)
        print("Training cost=", training_cost, "W=", t_w, "b=", t_b, '\n')

        raw_test_dataset = library.data_processing(test_data_path)
        X_test_d = pd.DataFrame(raw_test_dataset[['TEMP']]).to_numpy(
        )  # Change the variables here to train using different values
        Y_test_d = pd.DataFrame(raw_test_dataset[['PM2.5']]).to_numpy()
        print("Testing... (L2 loss Comparison)")
        testing_cost = sess.run(tf.reduce_sum(tf.pow(y_pred - Y, 2)) /
                                (2 * X_test_d.shape[0]),
                                feed_dict={
                                    X: X_test_d,
                                    Y: Y_test_d
                                })
        print("Testing cost=", testing_cost)
        print("Absolute l2 loss difference:",
              abs(training_cost - testing_cost))
# now declare the output data placeholder - 10 digits
y = tf.placeholder(tf.float32, [None, 47], name='y')

# now declare the weights connecting the input to the hidden layer
W1 = tf.Variable(tf.random_normal([784, 300], stddev=0.03), name='W1')
b1 = tf.Variable(tf.random_normal([300]), name='b1')
# and the weights connecting the hidden layer to the output layer
W2 = tf.Variable(tf.random_normal([300, 47], stddev=0.03), name='W2')
b2 = tf.Variable(tf.random_normal([47]), name='b2')


saver = tf.train.Saver()


# calculate the output of the hidden layer
hidden_out = tf.add(tf.matmul(x, W1), b1)
hidden_out = tf.nn.relu(hidden_out)

# now calculate the hidden layer output - in this case, let's use a softmax activated
# output layer
y_ = tf.nn.softmax(tf.add(tf.matmul(hidden_out, W2), b2))
tf.identity(y_, name="y_")

y_clipped = tf.clip_by_value(y_, 1e-10, 0.9999999)
cross_entropy = -tf.reduce_mean(tf.reduce_sum(y * tf.log(y_clipped) + (1 - y) * tf.log(1 - y_clipped), axis=1))

# add an optimiser
optimiser = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy)

# finally setup the initialisation operator
init_op = tf.global_variables_initializer()
print("sami")
print("W=", W)

#Now we will define the hyperparameters of the model,
#the Learning Rate and the number of Epochs.
learning_rate = 0.01
print("learning_rate", learning_rate)
training_epochs = 1000
print("training_epochs", training_epochs)

#Now, we will be building the Hypothesis, the Cost Function,
#and the Optimizer. We won’t be implementing the Gradient Descent Optimizer
#manually since it is built inside Tensorflow. After that, we will be initializing the Variables.

# Hypothesis
y_pred = tf.add(tf.multiply(X, W), b)
print("y_pred=", y_pred)

# Mean Squared Error Cost Function
cost = tf.reduce_sum(tf.pow(y_pred - Y, 2)) / (2 * n)
print("cost=", cost)

# Gradient Descent Optimizer
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
print("optimizer=", optimizer)

# Global Variables Initializer
init = tf.global_variables_initializer()

#Now we will begin the training process inside a Tensorflow Session.
예제 #19
0
def XceptionModel(input_image, num_classes, is_training = False, data_format='channels_last', name_prefix='', use_bn=True):
    bn_axis = -1 if data_format == 'channels_last' else 1

    
        
    
    # Entry Flow
    inputs = tf.layers.conv2d(input_image, 32, (3, 3), use_bias=False, name=name_prefix+'block1_conv1', strides=(2, 2),
                padding='valid', data_format=data_format, activation=None,
                kernel_initializer=tf.initializers.glorot_uniform(),
                bias_initializer=tf.zeros_initializer())
    
    
    inputs =  batch_norm_(inputs, name=name_prefix+'block1_conv1_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
    
#     inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block1_conv1_bn', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
    inputs = tf.nn.relu(inputs, name=name_prefix+'block1_conv1_act')

    inputs = tf.layers.conv2d(inputs, 64, (3, 3), use_bias=False, name=name_prefix+'block1_conv2', strides=(1, 1),
                padding='valid', data_format=data_format, activation=None,
                kernel_initializer=tf.initializers.glorot_uniform(),
                bias_initializer=tf.zeros_initializer())
    
    inputs =  batch_norm_(inputs, name=name_prefix+'block1_conv2_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block1_conv2_bn', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
    inputs = tf.nn.relu(inputs, name=name_prefix+'block1_conv2_act')

    residual = tf.layers.conv2d(inputs, 128, (1, 1), use_bias=False, name=name_prefix+'conv2d_1', strides=(2, 2),
                padding='same', data_format=data_format, activation=None,
                kernel_initializer=tf.initializers.glorot_uniform(),
                bias_initializer=tf.zeros_initializer())
    
    residual =  batch_norm_(residual, name=name_prefix+'batch_normalization_1', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_1', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)

    inputs = tf.layers.separable_conv2d(inputs, 128, (3, 3),
                        strides=(1, 1), padding='same',
                        data_format=data_format,
                        activation=None, use_bias=False,
                        depthwise_initializer=tf.initializers.glorot_uniform(),
                        pointwise_initializer=tf.initializers.glorot_uniform(),
                        bias_initializer=tf.zeros_initializer(),
                        name=name_prefix+'block2_sepconv1', reuse=None)
    inputs =  batch_norm_(inputs, name=name_prefix+'block1_sepconv1_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block2_sepconv1_bn', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)

    inputs = relu_separable_bn_block(inputs, 128, name_prefix+'block2_sepconv2', is_training, data_format, use_bn=use_bn)

    inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
                                    padding='same', data_format=data_format,
                                    name=name_prefix+'block2_pool')

    inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_0')
    residual = tf.layers.conv2d(inputs, 128, (1, 1), use_bias=False, name=name_prefix+'conv2d_2', strides=(2, 2),
                padding='same', data_format=data_format, activation=None,
                kernel_initializer=tf.initializers.glorot_uniform(),
                bias_initializer=tf.zeros_initializer())
    
    residual =  batch_norm_(residual, name=name_prefix+'batch_normalization_2', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_2', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)

    inputs = relu_separable_bn_block(inputs, 128, name_prefix+'block3_sepconv1', is_training, data_format, use_bn=use_bn)
    inputs = relu_separable_bn_block(inputs, 128, name_prefix+'block3_sepconv2', is_training, data_format, use_bn=use_bn)

    inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
                                    padding='same', data_format=data_format,
                                    name=name_prefix+'block3_pool')
    inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_1')

    residual = tf.layers.conv2d(inputs, 256, (1, 1), use_bias=False, name=name_prefix+'conv2d_3', strides=(2, 2),
                padding='same', data_format=data_format, activation=None,
                kernel_initializer=tf.initializers.glorot_uniform(),
                bias_initializer=tf.zeros_initializer())
    residual =  batch_norm_(residual, name=name_prefix+'batch_normalization_3', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_3', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)

    inputs = relu_separable_bn_block(inputs, 256, name_prefix+'block4_sepconv1', is_training, data_format, use_bn=use_bn)
    inputs = relu_separable_bn_block(inputs, 256, name_prefix+'block4_sepconv2', is_training, data_format, use_bn=use_bn)

    inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
                                    padding='same', data_format=data_format,
                                    name=name_prefix+'block4_pool')
    inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_2')
    # Middle Flow
    for index in range(8):
        residual = inputs
        prefix = name_prefix+'block' + str(index + 5)

        inputs = relu_separable_bn_block(inputs, 256, prefix + '_sepconv1', is_training, data_format, use_bn=use_bn)
        inputs = relu_separable_bn_block(inputs, 256, prefix + '_sepconv2', is_training, data_format, use_bn=use_bn)
        inputs = relu_separable_bn_block(inputs, 256, prefix + '_sepconv3', is_training, data_format, use_bn=use_bn)
        inputs = tf.add(inputs, residual, name=prefix + '_residual_add')
    # Exit Flow
    residual = tf.layers.conv2d(inputs, 512, (1, 1), use_bias=False, name=name_prefix+'conv2d_4', strides=(2, 2),
                padding='same', data_format=data_format, activation=None,
                kernel_initializer=tf.initializers.glorot_uniform(),
                bias_initializer=tf.zeros_initializer())
    residual =  batch_norm_(residual, name=name_prefix+'batch_normalization_4', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     residual = tf.layers.batch_normalization(residual, momentum=BN_MOMENTUM, name=name_prefix+'batch_normalization_4', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)

    inputs = relu_separable_bn_block(inputs, 512, name_prefix+'block13_sepconv1', is_training, data_format, use_bn=use_bn)
    inputs = relu_separable_bn_block(inputs, 512, name_prefix+'block13_sepconv2', is_training, data_format, use_bn=use_bn)

    inputs = tf.layers.max_pooling2d(inputs, pool_size=(3, 3), strides=(2, 2),
                                    padding='same', data_format=data_format,
                                    name=name_prefix+'block13_pool')
    inputs = tf.add(inputs, residual, name=name_prefix+'residual_add_3')

    inputs = tf.layers.separable_conv2d(inputs, 728, (3, 3),
                        strides=(1, 1), padding='same',
                        data_format=data_format,
                        activation=None, use_bias=False,
                        depthwise_initializer=tf.initializers.glorot_uniform(),
                        pointwise_initializer=tf.initializers.glorot_uniform(),
                        bias_initializer=tf.zeros_initializer(),
                        name=name_prefix+'block14_sepconv1', reuse=None)
    inputs =  batch_norm_(inputs, name=name_prefix+'block14_sepconv1_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)
#     inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block14_sepconv1_bn', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
    inputs = tf.nn.relu(inputs, name=name_prefix+'block14_sepconv1_act')

    inputs = tf.layers.separable_conv2d(inputs, 728, (3, 3),
                        strides=(1, 1), padding='same',
                        data_format=data_format,
                        activation=None, use_bias=False,
                        depthwise_initializer=tf.initializers.glorot_uniform(),
                        pointwise_initializer=tf.initializers.glorot_uniform(),
                        bias_initializer=tf.zeros_initializer(),
                        name=name_prefix+'block14_sepconv2', reuse=None)
    inputs =  batch_norm_(inputs, name=name_prefix+'block14_sepconv2_bn', axis=bn_axis,training=is_training,reuse=None, use_bn=use_bn)

#     inputs = tf.layers.batch_normalization(inputs, momentum=BN_MOMENTUM, name=name_prefix+'block14_sepconv2_bn', axis=bn_axis,
#                             epsilon=BN_EPSILON, training=is_training, reuse=None, fused=USE_FUSED_BN)
    inputs = tf.nn.relu(inputs, name=name_prefix+'block14_sepconv2_act')

    if data_format == 'channels_first':
        channels_last_inputs = tf.transpose(inputs, [0, 2, 3, 1])
    else:
        channels_last_inputs = inputs

    inputs = tf.layers.average_pooling2d(inputs, pool_size = reduced_kernel_size_for_small_input(channels_last_inputs, [10, 10]), strides = 1, padding='valid', data_format=data_format, name=name_prefix+'avg_pool')

    if data_format == 'channels_first':
        inputs = tf.squeeze(inputs, axis=[2, 3])
    else:
        inputs = tf.squeeze(inputs, axis=[1, 2])

    outputs = tf.layers.dense(inputs, num_classes,
                            activation=tf.nn.softmax, use_bias=True,
                            kernel_initializer=tf.initializers.glorot_uniform(),
                            bias_initializer=tf.zeros_initializer(),
                            name=name_prefix+'dense', reuse=None)

    return outputs
            damping_factor.copy(), name="damping_factor".format(i))

# the decoding neural network of Z=16
Z = 16
xa = tf.placeholder(tf.float32, shape=[batch_size, N, Z], name='xa')
ya = tf.placeholder(tf.float32, shape=[batch_size, N * Z], name='ya')
xa_input = tf.transpose(xa, [0, 2, 1])
net_dict["LLRa{0}".format(0)] = tf.zeros((batch_size, Z, sum_edge),
                                         dtype=tf.float32)
net_dict["infoM_lastlayera{0}".format(0)] = tf.zeros((batch_size, Z, sum_edge),
                                                     dtype=tf.float32)
for i in range(0, iters_max, 1):
    #variable node update
    x0 = tf.matmul(xa_input, W_skipconn2even)
    x1 = tf.matmul(net_dict["LLRa{0}".format(i)], W_odd2even)
    x2_3 = tf.add(x0, x1)
    x2 = tf.add(
        tf.multiply(x2_3, 1 - net_dict["damping_factor{0}".format(i)]),
        tf.multiply(net_dict["infoM_lastlayera{0}".format(i)],
                    net_dict["damping_factor{0}".format(i)]))
    net_dict["infoM_lastlayera{0}".format(i + 1)] = x2
    x2 = tf.transpose(x2, [0, 2, 1])
    x2 = tf.reshape(x2, [batch_size, neurons_per_odd_layer * Z])
    x2 = tf.matmul(x2, Lift_Matrix1[0].transpose())
    x2 = tf.reshape(x2, [batch_size, neurons_per_odd_layer, Z])
    x2 = tf.transpose(x2, [0, 2, 1])
    x_tile = tf.tile(x2, multiples=[1, 1, neurons_per_odd_layer])
    W_input_reshape = tf.reshape(W_even2odd.transpose(), [-1])
    #check node update
    x_tile_mul = tf.multiply(x_tile, W_input_reshape)
    x2_1 = tf.reshape(
예제 #21
0
def inference(images, compression_obj):
  """Build the CIFAR-10 model.

  Args:
    images: Images returned from distorted_inputs() or inputs().
    compression_obj: The compression object to use for compressing matrices in
      the main graph.

  Returns:
    Logits.
  """
  # We instantiate all variables using tf.compat.v1.get_variable() instead of
  # tf.Variable() in order to share variables across multiple GPU training runs.
  # If we only ran this model on a single GPU, we could simplify this function
  # by replacing all instances of tf.compat.v1.get_variable() with
  # tf.Variable().
  #
  # While instantiating conv and local layers, we add mask and threshold
  # variables to the layer by calling the pruning.apply_mask() function.
  # Note that the masks are applied only to the weight tensors
  # conv1
  with tf.variable_scope('conv1') as scope:
    kernel = _variable_with_weight_decay(
        'weights', shape=[5, 5, 3, 64], stddev=5e-2, wd=0.0)

    conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
    biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.0))
    pre_activation = tf.nn.bias_add(conv, biases)
    conv1 = tf.nn.relu(pre_activation, name=scope.name)
    _activation_summary(conv1)

  # pool1
  pool1 = tf.nn.max_pool(
      conv1,
      ksize=[1, 3, 3, 1],
      strides=[1, 2, 2, 1],
      padding='SAME',
      name='pool1')
  # norm1
  norm1 = tf.nn.lrn(
      pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')

  # conv2
  with tf.variable_scope('conv2') as scope:
    kernel = _variable_with_weight_decay(
        'weights', shape=[5, 5, 64, 64], stddev=5e-2, wd=0.0)

    conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
    biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.1))
    pre_activation = tf.nn.bias_add(conv, biases)
    conv2 = tf.nn.relu(pre_activation, name=scope.name)
    _activation_summary(conv2)

  # norm2
  norm2 = tf.nn.lrn(
      conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
  # pool2
  pool2 = tf.nn.max_pool(
      norm2,
      ksize=[1, 3, 3, 1],
      strides=[1, 2, 2, 1],
      padding='SAME',
      name='pool2')

  # local3
  with tf.variable_scope('local3') as scope:
    # Move everything into depth so we can perform a single matrix multiply.
    reshape = tf.reshape(pool2, [BATCH_SIZE, -1])
    dim = reshape.get_shape()[1].value
    weights = _variable_with_weight_decay(
        'weights', shape=[dim, 384], stddev=0.04, wd=0.004)
    biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1))

    local3 = tf.nn.relu(
        tf.matmul(reshape, compression_obj.apply_compression(weights, scope)) +
        biases,
        name=scope.name)
    _activation_summary(local3)

  # local4
  with tf.variable_scope('local4') as scope:
    weights = _variable_with_weight_decay(
        'weights', shape=[384, 192], stddev=0.04, wd=0.004)
    biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1))

    local4 = tf.nn.relu(
        tf.matmul(local3, compression_obj.apply_compression(weights, scope)) +
        biases,
        name=scope.name)
    _activation_summary(local4)

  # linear layer(WX + b),
  # We don't apply softmax here because
  # tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
  # and performs the softmax internally for efficiency.
  with tf.variable_scope('softmax_linear') as scope:
    weights = _variable_with_weight_decay(
        'weights', [192, NUM_CLASSES], stddev=1 / 192.0, wd=0.0)
    biases = _variable_on_cpu('biases', [NUM_CLASSES],
                              tf.constant_initializer(0.0))

    softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)
    _activation_summary(softmax_linear)

  return softmax_linear
예제 #22
0
# _*_ coding utf-8 _*_
# Author:94342
# Time:  2020/9/1717:09
# File:  New03.py
# Engine:PyCharm


import tensorflow.compat.v1 as tf


if __name__ == '__main__':
    tf.compat.v1.disable_eager_execution()

    v1 = tf.Variable(0, name='counter')
    one = tf.constant(1)
    temp = tf.add(v1, one)
    process = tf.assign(v1, temp)
    init = tf.initialize_all_variables()

    with tf.Session() as sess:
        sess.run(init)
        print(sess.run(v1))
        for i in range(3):
            sess.run(process)
            print(sess.run(v1))
예제 #23
0
# In[1]:

# Task 1 - 1
# TIES 4911
# Toni Pikkarainen
# 14.1.2020
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

# In[2]:

# Constants
# Lecture02, slides 11-13
a = tf.constant(5)
b = tf.constant(2)
add_op = tf.add(a, b)
mul_op = tf.multiply(b, add_op)
pow_op = tf.pow(mul_op, b)

# In[3]:

# Variables
# Lecture02, slides 15-17
var1 = tf.Variable(2, name="scalar1")
var2 = tf.Variable(3, name="scalar2")
assign_op = var2.assign(10)

# In[4]:

# Placeholders
# Lecture02, slides 19-20
예제 #24
0
def dilated_layer(name,
                  input_batch,
                  dilation,
                  dilation_width,
                  dilation_units,
                  residual=False,
                  dense_residual=False,
                  to_batch_norm=False):
    '''Create a dilation layer:
        INPUT:
        name: must be unique for graph purpose
        input_batch: 3D input tensor batch, length, channels/width
        Current implementation keeps the channels/hidden units intakt
        dialton: dialtion rate to apply
        dilation_width: with of the dilation filter (only 2 supported?)
        dilation_units: dilation_units or channels
        residual: True/False --> select if to propagate residual in that layer/stack
        to_batch_norm: True/False select of to perform batch norm at every layer
        RETURNS:
        3D tensor batch, length-dilation rate, channels width'''
    with tf.name_scope(name):
        # get shapes
        channels = input_batch.get_shape().as_list()[2]
        # create variables
        dilation_weights = create_variable(
            'dilation_weights', [dilation_width, channels, dilation_units])
        dilation_biases = create_bias_variable('dilation_biases',
                                               [dilation_units])
        gate_weights = create_variable(
            'gate_weights', [dilation_width, channels, dilation_units])
        gate_biases = create_bias_variable('gate_biases', [dilation_units])
        # redisual and skip
        if residual == True:
            if dense_residual == True:
                dense_weights = create_variable(
                    'dense_weights',
                    [dilation_units, channels, dilation_units])
                dense_biases = create_bias_variable('dense_biases',
                                                    [dilation_units])
                # skip_weights = create_variable('skip_weights', [1, dilation_units, skip_units])
                # skip_biases = create_bias_variable('skip_biases', [skip_units])

        # define convolutional steps
        dilated = tf.add(
            dilated_conv(input_batch, dilation_weights, dilation=dilation),
            dilation_biases)
        gated = tf.add(
            dilated_conv(input_batch, gate_weights, dilation=dilation),
            gate_biases)
        dilated_gated = tf.tanh(dilated) * tf.sigmoid(gated)

        if residual == True:
            # if dense residual connection desired make a 1x1 convolutiion before adding
            if dense_residual == True:
                # 1x1 dense convolution for residual
                transformed = tf.nn.conv1d(dilated_gated,
                                           dense_weights,
                                           stride=1,
                                           padding="SAME",
                                           name="dense")
                transformed = transformed + dense_biases
                # add up residual to 1x1 transformed output
                out = input_batch + transformed
            else:
                # else just add input_batch shortcut to dilated/gated
                out = input_batch + dilated_gated
        else:
            # else dilated gated is out
            out = dilated_gated
        # # The 1x1 conv to produce the skip output
        # skip_cut = tf.shape(out)[1] - output_width
        # out_skip = tf.slice(out, [0, skip_cut, 0], [-1, -1, -1])
        # weights_skip = variables['skip']
        # skip_contribution = tf.nn.conv1d(
        #     out_skip, weights_skip, stride=1, padding="SAME", name="skip")
        # # batch norm
        if to_batch_norm == True:
            out = tf.layers.batch_normalization(out)

        # make summary histograms of weights
        tf.summary.histogram(name + '_dilation_weights', dilation_weights)
        tf.summary.histogram(name + '_dilation_biases', dilation_biases)
        tf.summary.histogram(name + '_gate_weights', gate_weights)
        tf.summary.histogram(name + '_gate_biases', gate_biases)
        if dense_residual == True:
            tf.summary.histogram(name + '_dense_weights', dense_weights)
            tf.summary.histogram(name + '_dense_biases', dense_biases)
        # tf.summary.histogram(name + '_skip_weights', skip_weights)
        # tf.summary.histogram(name + '_skip_biases', skip_biases)

        return out
예제 #25
0
def inference(seqs, conv_layers, hidden_units_scheme, kernel_width_scheme,
              max_pool_scheme, dilation_scheme, dilation_units, dilation_width,
              dilation_residual, dilation_residual_dense, dilation_batch_norm,
              num_classes, batch_size, keep_prob_inner, keep_prob_outer,
              seed_weights, seed_scheme, seed_weights_list,
              report_conv_hidden_state):
    """INFERENCE
    Args:

    Returns:
        regression score or hidden representation after convolutional but before dilated convolutional layer
    """

    print('seqs shape')
    print(seqs.get_shape().as_list())

    current_layer = seqs

    # Convolutional Stack with Max Pooling =====================================
    # run an inital dilated layer with dilation 1 to map to the dilational unit output
    with tf.name_scope('Convolutional_stack'):
        for i in range(conv_layers):
            j = i + 1
            k = i * 2
            if seed_weights and seed_scheme[i] == 1:
                weights_load_string = 'arr_' + str(k)
                biases_load_string = 'arr_' + str(k + 1)
                print('Pre-seeding Layer: ' + str(j))
                current_layer = convolutional_layer(
                    'conv_layer{}'.format(j),
                    current_layer,
                    hidden_units_scheme[i],
                    kernel_width_scheme[i],
                    max_pool_scheme[i],
                    keep_prob_inner,
                    True,
                    seed_weights_list[weights_load_string],
                    seed_weights_list[biases_load_string],
                    to_batch_norm=False)
            else:
                current_layer = convolutional_layer('conv_layer{}'.format(j),
                                                    current_layer,
                                                    hidden_units_scheme[i],
                                                    kernel_width_scheme[i],
                                                    max_pool_scheme[i],
                                                    keep_prob_inner,
                                                    False,
                                                    "dummy",
                                                    "dummy",
                                                    to_batch_norm=False)
            print('Conv %s shape' % j)
            print(current_layer.get_shape().as_list())

    # Report only HIdden STate afte r Conv layers (optional) ===================
    if report_conv_hidden_state:
        ''''only report hte hidden state after the convolutional stacks'''
        hidden_conv_state = current_layer
        return (hidden_conv_state)

    # Dilational Layers stack ==================================================
    # run an inital dilated layer with dilation 1 to map to the dilational unit output
    with tf.name_scope('dilated_stack'):
        current_layer = dilated_layer('dilated_layer1',
                                      current_layer,
                                      1,
                                      dilation_width,
                                      dilation_units,
                                      residual=dilation_residual,
                                      dense_residual=dilation_residual_dense,
                                      to_batch_norm=dilation_batch_norm)
        print('Dilated shape')
        print(current_layer.get_shape().as_list())
        for i, dilation in enumerate(dilation_scheme):
            i = i + 1  # skipping 0 count as this is pre-established
            current_layer = dilated_layer(
                'dilated_layer{}'.format(i),
                current_layer,
                dilation,
                dilation_width,
                dilation_units,
                residual=dilation_residual,
                dense_residual=dilation_residual_dense,
                to_batch_norm=dilation_batch_norm)

            print('Dilated shape')
            print(current_layer.get_shape().as_list())

    # reshape for FC layer
    with tf.name_scope('reshape_layer'):
        fully_connected_width = current_layer.get_shape().as_list(
        )[1] * dilation_units
        current_layer = tf.reshape(current_layer,
                                   [batch_size, fully_connected_width])
        print('fully connection reshaped')
        print(current_layer.get_shape().as_list())

    # Final full connection(s) into logits
    with tf.name_scope('final_dense'):
        weights = create_variable('weights',
                                  [fully_connected_width, num_classes])
        biases = create_bias_variable('biases', [num_classes])
        regression_score = tf.add(tf.matmul(current_layer, weights), biases)
        print('Regression score shape')
        print(regression_score.get_shape().as_list())
        _activation_summary(regression_score)
        tf.summary.histogram('final_dense_weights', weights)

    # return regression_score, into_dilation
    return regression_score
예제 #26
0
    def _assemble_graph(self):
        self._log('Parsing dataset...')
        self._graph_parse_data()

        self._log('Creating layer parameters...')
        self._add_layers_to_graph()

        self._log('Assembling graph...')

        x, y = tf.train.shuffle_batch([self._train_images, self._train_labels],
                                      batch_size=self._batch_size,
                                      num_threads=self._num_threads,
                                      capacity=self._queue_capacity,
                                      min_after_dequeue=self._batch_size)

        # Reshape input to the expected image dimensions
        x = tf.reshape(x,
                       shape=[
                           -1, self._image_height, self._image_width,
                           self._image_depth
                       ])

        # Run the network operations
        xx = self.forward_pass(x, deterministic=False)

        # Define regularization cost
        if self._reg_coeff is not None:
            l2_cost = tf.squeeze(
                tf.reduce_sum([
                    layer.regularization_coefficient *
                    tf.nn.l2_loss(layer.weights) for layer in self._layers
                    if isinstance(layer, layers.fullyConnectedLayer)
                ]))
        else:
            l2_cost = 0.0

        # Define cost function
        if self._loss_fn == 'l1':
            l1_loss = tf.reduce_mean(tf.abs(tf.subtract(xx, y)))
            gt = tf.reduce_sum(y, axis=[1, 2, 3]) / (32**2.0)
            pr = tf.reduce_sum(xx, axis=[1, 2, 3]) / (32**2.0)
            accuracy = tf.reduce_mean(tf.abs(gt - pr))
            self._graph_ops['cost'] = tf.squeeze(tf.add(l1_loss, l2_cost))
            self._graph_ops['accuracy'] = accuracy

        # Set the optimizer and get the gradients from it
        gradients, variables, global_grad_norm = self._graph_add_optimizer()

        # Calculate validation and test accuracy
        if self._testing:
            x_test, self._graph_ops['y_test'] = tf.train.batch(
                [self._test_images, self._test_labels],
                batch_size=self._batch_size,
                num_threads=self._num_threads,
                capacity=self._queue_capacity)
            x_test = tf.reshape(x_test,
                                shape=[
                                    -1, self._image_height, self._image_width,
                                    self._image_depth
                                ])
        if self._validation:
            x_val, self._graph_ops['y_val'] = tf.train.batch(
                [self._val_images, self._val_labels],
                batch_size=self._batch_size,
                num_threads=self._num_threads,
                capacity=self._queue_capacity)
            x_val = tf.reshape(x_val,
                               shape=[
                                   -1, self._image_height, self._image_width,
                                   self._image_depth
                               ])

        if self._testing:
            self._graph_ops['x_test_predicted'] = self.forward_pass(
                x_test, deterministic=True)
        if self._validation:
            self._graph_ops['x_val_predicted'] = self.forward_pass(
                x_val, deterministic=True)

        if self._testing:
            if self._loss_fn == 'l1':
                self._graph_ops['test_losses'] = tf.reduce_mean(
                    tf.abs(
                        tf.subtract(self._graph_ops['y_test'],
                                    self._graph_ops['x_test_predicted'])))
                gt_test = tf.reduce_sum(self._graph_ops['y_test'],
                                        axis=[1, 2, 3]) / (32**2.0)
                pr_test = tf.reduce_sum(self._graph_ops['x_test_predicted'],
                                        axis=[1, 2, 3]) / (32**2.0)
                self._graph_ops['gt_test'] = gt_test
                self._graph_ops['pr_test'] = pr_test
                self._graph_ops['test_accuracy'] = tf.reduce_mean(
                    tf.abs(gt_test - pr_test))
        if self._validation:
            if self._loss_fn == 'l1':
                self._graph_ops['val_losses'] = tf.reduce_mean(
                    tf.abs(
                        tf.subtract(self._graph_ops['y_val'],
                                    self._graph_ops['x_val_predicted'])))
                gt_val = tf.reduce_sum(self._graph_ops['y_val'],
                                       axis=[1, 2, 3]) / (32**2.0)
                pr_val = tf.reduce_sum(self._graph_ops['x_val_predicted'],
                                       axis=[1, 2, 3]) / (32**2.0)
                self._graph_ops['val_accuracy'] = tf.reduce_mean(
                    tf.abs(gt_val - pr_val))

            # Epoch summaries for Tensorboard
            if self._tb_dir is not None:
                self._graph_tensorboard_summary(l2_cost, gradients, variables,
                                                global_grad_norm)
예제 #27
0
    def compute_gradients(self,
                          loss,
                          var_list,
                          gate_gradients=GATE_OP,
                          aggregation_method=None,
                          colocate_gradients_with_ops=False,
                          grad_loss=None,
                          gradient_tape=None):
      if callable(loss):
        # TF is running in Eager mode, check we received a vanilla tape.
        if not gradient_tape:
          raise ValueError('When in Eager mode, a tape needs to be passed.')

        vector_loss = loss()
        if self._num_microbatches is None:
          self._num_microbatches = tf.shape(input=vector_loss)[0]
        sample_state = self._dp_sum_query.initial_sample_state(var_list)
        microbatches_losses = tf.reshape(vector_loss,
                                         [self._num_microbatches, -1])
        sample_params = (
            self._dp_sum_query.derive_sample_params(self._global_state))

        def process_microbatch(i, sample_state):
          """Process one microbatch (record) with privacy helper."""
          microbatch_loss = tf.reduce_mean(
              input_tensor=tf.gather(microbatches_losses, [i]))
          grads = gradient_tape.gradient(microbatch_loss, var_list)
          sample_state = self._dp_sum_query.accumulate_record(
              sample_params, sample_state, grads)
          return sample_state

        for idx in range(self._num_microbatches):
          sample_state = process_microbatch(idx, sample_state)

        grad_sums, self._global_state = (
            self._dp_sum_query.get_noised_result(
                sample_state, self._global_state))

        def normalize(v):
          return v / tf.cast(self._num_microbatches, tf.float32)

        final_grads = tf.nest.map_structure(normalize, grad_sums)

        grads_and_vars = list(zip(final_grads, var_list))
        return grads_and_vars

      else:
        # TF is running in graph mode, check we did not receive a gradient tape.
        if gradient_tape:
          raise ValueError('When in graph mode, a tape should not be passed.')

        # Note: it would be closer to the correct i.i.d. sampling of records if
        # we sampled each microbatch from the appropriate binomial distribution,
        # although that still wouldn't be quite correct because it would be
        # sampling from the dataset without replacement.
        if self._num_microbatches is None:
          self._num_microbatches = tf.shape(input=loss)[0]

        microbatches_losses = tf.reshape(loss, [self._num_microbatches, -1])
        sample_params = (
            self._dp_sum_query.derive_sample_params(self._global_state))

        def process_microbatch(i, sample_state):
          """Process one microbatch (record) with privacy helper."""
          grads, _ = zip(*super(cls, self).compute_gradients(
              tf.reduce_mean(input_tensor=tf.gather(
                  microbatches_losses, [i])), var_list, gate_gradients,
              aggregation_method, colocate_gradients_with_ops, grad_loss))
          grads_list = [
              g if g is not None else tf.zeros_like(v)
              for (g, v) in zip(list(grads), var_list)
          ]
          sample_state = self._dp_sum_query.accumulate_record(
              sample_params, sample_state, grads_list)
          return sample_state

        if var_list is None:
          var_list = (
              tf.compat.v1.trainable_variables() + tf.compat.v1.get_collection(
                  tf.compat.v1.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))

        sample_state = self._dp_sum_query.initial_sample_state(var_list)

        if self._unroll_microbatches:
          for idx in range(self._num_microbatches):
            sample_state = process_microbatch(idx, sample_state)
        else:
          # Use of while_loop here requires that sample_state be a nested
          # structure of tensors. In general, we would prefer to allow it to be
          # an arbitrary opaque type.
          cond_fn = lambda i, _: tf.less(i, self._num_microbatches)
          body_fn = lambda i, state: [tf.add(i, 1), process_microbatch(i, state)]  # pylint: disable=line-too-long
          idx = tf.constant(0)
          _, sample_state = tf.while_loop(
              cond=cond_fn, body=body_fn, loop_vars=[idx, sample_state])

        grad_sums, self._global_state = (
            self._dp_sum_query.get_noised_result(
                sample_state, self._global_state))

        def normalize(v):
          return tf.truediv(v, tf.cast(self._num_microbatches, tf.float32))

        final_grads = tf.nest.map_structure(normalize, grad_sums)

        return list(zip(final_grads, var_list))
potentials = []
valid_potentials = []
wave_functions = []
valid_functions = []

sess = tf.Session()
sess.run(init)
for i in range(npots):
    if i % 10 == 0:
        print(str((100. * i) / npots) + '% complete')
    for j in range(3):
        vofx = generate_potential(j, (1. * i) / npots)
        energy = tf.reduce_mean(
            tf.subtract(
                tf.multiply(tf.square(psi), tf.add(vofx, 1. * bins * bins)),
                tf.multiply(tf.multiply(tf.add(psil, psir), psi),
                            0.5 * bins * bins)))
        training = optim.minimize(energy)
        sess.run(reinit)
        for t in range(20000):
            sess.run(training)
            sess.run(renorm)
        if i % validnth == 0:
            valid_potentials.append(vofx)
            valid_functions.append(sess.run(psi).tolist())
        else:
            potentials.append(vofx)
            wave_functions.append(sess.run(psi).tolist())

with open('test_potentials' + str(seed) + '.csv', 'w') as f:
def convolutional_block(X_input,
                        kernel_size,
                        in_filter,
                        out_filters,
                        stage,
                        block,
                        training,
                        stride=2):
    """
        Implementation of the convolutional block as defined in Figure 4

        Arguments:
        X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
        kernel_size -- integer, specifying the shape of the middle CONV's window for the main path
        filters -- python list of integers, defining the number of filters in the CONV layers of the main path
        stage -- integer, used to name the layers, depending on their position in the network
        block -- string/character, used to name the layers, depending on their position in the network
        training -- train or test
        stride -- Integer, specifying the stride to be used

        Returns:
        X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)
        """

    # defining name basis
    block_name = 'res' + str(stage) + block
    with tf.variable_scope(block_name):
        f1, f2, f3 = out_filters

        x_shortcut = X_input
        #first
        W_conv1 = weight_variable([1, 1, in_filter, f1])
        X = tf.nn.conv2d(X_input,
                         W_conv1,
                         strides=[1, stride, stride, 1],
                         padding='VALID')
        X = tf.layers.batch_normalization(X, axis=3, training=training)
        X = tf.nn.relu(X)

        #second
        W_conv2 = weight_variable([kernel_size, kernel_size, f1, f2])
        X = tf.nn.conv2d(X, W_conv2, strides=[1, 1, 1, 1], padding='SAME')
        X = tf.layers.batch_normalization(X, axis=3, training=training)
        X = tf.nn.relu(X)

        #third
        W_conv3 = weight_variable([1, 1, f2, f3])
        X = tf.nn.conv2d(X, W_conv3, strides=[1, 1, 1, 1], padding='VALID')
        X = tf.layers.batch_normalization(X, axis=3, training=training)

        #shortcut path
        W_shortcut = weight_variable([1, 1, in_filter, f3])
        x_shortcut = tf.nn.conv2d(x_shortcut,
                                  W_shortcut,
                                  strides=[1, stride, stride, 1],
                                  padding='VALID')

        #final
        add = tf.add(x_shortcut, X)
        add_result = tf.nn.relu(add)

    return add_result
예제 #30
0
    def prepare_processing_graph(self, model_settings):
        """Builds a TensorFlow graph to apply the input distortions.

    Creates a graph that loads a WAVE file, decodes it, scales the volume,
    shifts it in time, adds in background noise, calculates a spectrogram, and
    then builds an MFCC fingerprint from that.

    This must be called with an active TensorFlow session running, and it
    creates multiple placeholder inputs, and one output:

      - wav_filename_placeholder_: Filename of the WAV to load.
      - foreground_volume_placeholder_: How loud the main clip should be.
      - time_shift_padding_placeholder_: Where to pad the clip.
      - time_shift_offset_placeholder_: How much to move the clip in time.
      - background_data_placeholder_: PCM sample data for background noise.
      - background_volume_placeholder_: Loudness of mixed-in background.
      - mfcc_: Output 2D fingerprint of processed audio.

    Args:
      model_settings: Information about the current model being trained.
    """
        desired_samples = model_settings['desired_samples']
        channel_count = model_settings['channel_count']
        sample_rate = model_settings['sample_rate']
        self.foreground_data_placeholder_ = tf.placeholder(
            tf.float32, [desired_samples, channel_count])
        # Allow the audio sample's volume to be adjusted.
        self.foreground_volume_placeholder_ = tf.placeholder(tf.float32, [])
        scaled_foreground = tf.multiply(self.foreground_data_placeholder_,
                                        self.foreground_volume_placeholder_)
        # Shift the sample's start position, and pad any gaps with zeros.
        self.time_shift_padding_placeholder_ = tf.placeholder(tf.int32, [2, 2])
        self.time_shift_offset_placeholder_ = tf.placeholder(tf.int32, [2])
        padded_foreground = tf.pad(scaled_foreground,
                                   self.time_shift_padding_placeholder_,
                                   mode='CONSTANT')
        sliced_foreground = tf.slice(padded_foreground,
                                     self.time_shift_offset_placeholder_,
                                     [desired_samples, -1])
        # Mix in background noise.
        self.background_data_placeholder_ = tf.placeholder(
            tf.float32, [desired_samples, channel_count])
        self.background_volume_placeholder_ = tf.placeholder(tf.float32, [])
        background_mul = tf.multiply(self.background_data_placeholder_,
                                     self.background_volume_placeholder_)
        background_add = tf.add(background_mul, sliced_foreground)
        background_clamp = tf.clip_by_value(background_add, -1.0, 1.0)
        # Run the spectrogram and MFCC ops to get a 2D 'fingerprint' of the audio.
        self.waveform_ = background_clamp
        spectrograms = []
        for ichannel in range(channel_count):
            spectrograms.append(
                audio_ops.audio_spectrogram(
                    tf.slice(background_clamp, [0, ichannel], [-1, 1]),
                    window_size=model_settings['window_size_samples'],
                    stride=model_settings['window_stride_samples'],
                    magnitude_squared=True))
        self.spectrogram_ = tf.stack(spectrograms, -1)
        mfccs = []
        for ichannel in range(channel_count):
            mfccs.append(
                audio_ops.mfcc(
                    spectrograms[ichannel],
                    sample_rate,
                    upper_frequency_limit=model_settings['sample_rate'] // 2,
                    filterbank_channel_count=model_settings[
                        'filterbank_channel_count'],
                    dct_coefficient_count=model_settings[
                        'dct_coefficient_count']))
        self.mfcc_ = tf.stack(mfccs, -1)