def read_and_decode(filename_queue, batch_size):

    reader = tf.TFRecordReader()
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        # Defaults are not specified since both keys are required.
        features={
            'height':
            tf.FixedLenFeature([], tf.int64),
            'word_opinion':
            tf.FixedLenFeature([Hp.w_maxlen * 6],
                               dtype=tf.int64,
                               default_value=[-1] * Hp.w_maxlen * 6),
            'char_opinion':
            tf.FixedLenFeature([], tf.string)
        })

    char_opinion = tf.decode_raw(features['char_opinion'], tf.uint8)
    height = tf.cast(features['height'], tf.int32)
    word_opinion = tf.cast(features['word_opinion'], tf.int32)

    char_opinion = tf.reshape(char_opinion, tf.stack([6, Hp.c_maxlen]))
    word_opinion = tf.reshape(word_opinion, tf.stack([6, Hp.w_maxlen]))
    words, chars = tf.train.batch([word_opinion, char_opinion],
                                  batch_size=batch_size,
                                  capacity=3 * batch_size,
                                  num_threads=1)

    return (words, chars)
Example #2
0
def ner_accuracy(tensor, opt):
    r"""Returns accuracy of predictions.

    Args:
      tensor: A `Tensor`. Probability distributions or unscaled prediction scores.
      opt:
        target: A 'Tensor`. Labels.

    Returns:
      A `Tensor` of the same shape as `tensor`. Each value will be 1 if correct else 0. 

    For example,

    ```
    tensor = [[20.1, 18, -4.2], [0.04, 21.1, 31.3]]
    target = [[0, 1]]
    tensor.sg_accuracy(target=target) => [[ 1.  0.]]
    ```
    """
    assert opt.target is not None, 'target is mandatory.'
    opt += tf.sg_opt(k=1)

    # # calc accuracy
    out = tf.identity(tf.equal(tensor.sg_argmax() + 1, tf.cast(opt.target, tf.int64)).sg_float(), name='acc')
    # out = tf.identity(tf.nn.in_top_k(tensor, opt.target, opt.k).sg_float(), name='acc')

    # masking padding
    if opt.mask:
        out += tf.equal(opt.target, tf.zeros_like(opt.target)).sg_float()

    return out
Example #3
0
def rnn_classify(x, num_classes, is_test=False):
    with tf.sg_context(name='rnn_classify'):
        fw_cell = tf.nn.rnn_cell.MultiRNNCell(
            [lstm_cell(is_test) for _ in range(num_blocks)],
            state_is_tuple=True)
        bw_cell = tf.nn.rnn_cell.MultiRNNCell(
            [lstm_cell(is_test) for _ in range(num_blocks)],
            state_is_tuple=True)

        words_used_in_sent = tf.sign(
            tf.reduce_max(tf.abs(x), reduction_indices=2))
        length = tf.cast(
            tf.reduce_sum(words_used_in_sent, reduction_indices=1), tf.int32)

        outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell,
                                                     bw_cell,
                                                     x,
                                                     dtype=tf.float32,
                                                     sequence_length=length)
        output = tf.concat(outputs, 2).sg_reshape(shape=[-1, 2 * latent_dim])

        prediction = output.sg_dense(dim=num_classes, name='dense')
        res = tf.reshape(prediction,
                         [x.get_shape().as_list()[0], -1, num_classes])

    return res
Example #4
0
def ner_accuracy(tensor, opt):
    r"""Returns accuracy of predictions.

    Args:
      tensor: A `Tensor`. Probability distributions or unscaled prediction scores.
      opt:
        target: A 'Tensor`. Labels.

    Returns:
      A `Tensor` of the same shape as `tensor`. Each value will be 1 if correct else 0. 

    For example,

    ```
    tensor = [[20.1, 18, -4.2], [0.04, 21.1, 31.3]]
    target = [[0, 1]]
    tensor.sg_accuracy(target=target) => [[ 1.  0.]]
    ```
    """
    assert opt.target is not None, 'target is mandatory.'
    opt += tf.sg_opt(k=1)

    # # calc accuracy
    out = tf.identity(tf.equal(tensor.sg_argmax() + 1,
                               tf.cast(opt.target, tf.int64)).sg_float(),
                      name='acc')
    # out = tf.identity(tf.nn.in_top_k(tensor, opt.target, opt.k).sg_float(), name='acc')

    # masking padding
    if opt.mask:
        out += tf.equal(opt.target, tf.zeros_like(opt.target)).sg_float()

    return out
Example #5
0
def ner_cost(tensor, opt):
    one_hot_labels = tf.one_hot(opt.target - 1, opt.num_classes, dtype=tf.float32)
    cross_entropy = one_hot_labels * tf.log(tensor)
    cross_entropy = -tf.reduce_sum(cross_entropy, reduction_indices=2)

    mask = tf.sign(tf.abs(opt.target))

    cross_entropy *= tf.cast(mask, tf.float32)
    cross_entropy = tf.reduce_sum(cross_entropy, reduction_indices=1)

    length = tf.cast(tf.reduce_sum(tf.sign(opt.target), reduction_indices=1), tf.int32)
    cross_entropy /= tf.cast(length, tf.float32)

    out = tf.reduce_mean(cross_entropy, name='ner_cost')

    # add summary
    tf.sg_summary_loss(out, name=opt.name)

    return out
Example #6
0
def ner_cost(tensor, opt):
    one_hot_labels = tf.one_hot(opt.target - 1, opt.num_classes, dtype=tf.float32)
    cross_entropy = one_hot_labels * tf.log(tensor)
    cross_entropy = -tf.reduce_sum(cross_entropy, reduction_indices=2)

    mask = tf.sign(tf.reduce_max(tf.abs(one_hot_labels), reduction_indices=2))

    cross_entropy *= tf.cast(mask, tf.float32)
    cross_entropy = tf.reduce_sum(cross_entropy, reduction_indices=1)

    length = tf.cast(tf.reduce_sum(tf.sign(opt.target), reduction_indices=1), tf.int32)
    cross_entropy /= tf.cast(length, tf.float32)

    out = tf.reduce_mean(cross_entropy, name='ner_cost')

    # add summary
    tf.sg_summary_loss(out, name=opt.name)

    return out
Example #7
0
def testIt():
    data = raw
    positive = np.array(data.label_train) > 0
    x = tf.placeholder(tf.float32, [None, 4096])
    y = tf.placeholder(tf.float32)
    disc_real = discriminator(x)
    accuracy = tf.reduce_mean(
        tf.cast(tf.equal(tf.cast(disc_real > 0.5, "float"), y), tf.float32))
    np.set_printoptions(precision=3, suppress=True)
    with tf.Session() as sess:
        sess.run(
            tf.group(tf.global_variables_initializer(),
                     tf.sg_phase().assign(False)))
        # restore parameters
        tf.sg_restore(sess,
                      tf.train.latest_checkpoint('asset/train/gan'),
                      category=['generator', 'discriminator'])
        ans = sess.run(disc_real, feed_dict={x: np.array(data.test)})
        print np.sum(ans > 0.5)
        np.save('dm_bird.npy', ans)
Example #8
0
def sg_accuracy(tensor, opt):
    assert opt.target is not None, 'target is mandatory.'
    opt += tf.sg_opt(k=1)

    # # calc accuracy
    out = tf.identity(tf.equal(tensor.sg_argmax(),
                               tf.cast(opt.target, tf.int64)).sg_float(),
                      name='acc')
    # out = tf.identity(tf.nn.in_top_k(tensor, opt.target, opt.k).sg_float(), name='acc')

    return out
Example #9
0
    def iou(self, boxes1, boxes2):
        """
        Note: Modified from https://github.com/nilboy/tensorflow-yolo/blob/python2.7/yolo/net/yolo_net.py
        calculate ious
        Args:
          boxes1: 4-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL, 4]  ====> (x_center, y_center, w, h)
          boxes2: 1-D tensor [4] ===> (x_center, y_center, w, h)
        Return:
          iou: 3-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
        """
        boxes1 = tf.stack([
            boxes1[:, 0] - boxes1[:, 2] / 2, boxes1[:, 1] - boxes1[:, 3] / 2,
            boxes1[:, 0] + boxes1[:, 2] / 2, boxes1[:, 1] + boxes1[:, 3] / 2
        ])

        boxes2 = tf.stack([
            boxes2[:, 0] - boxes2[:, 2] / 2, boxes2[:, 1] - boxes2[:, 3] / 2,
            boxes2[:, 0] + boxes2[:, 2] / 2, boxes2[:, 1] + boxes2[:, 3] / 2
        ])

        #calculate the left up point

        lu = tf.maximum(boxes1[0:2], boxes2[0:2])
        rd = tf.minimum(boxes1[2:], boxes2[2:])

        #intersection
        intersection = rd - lu

        inter_square = tf.multiply(intersection[0], intersection[1])

        mask = tf.cast(intersection[0] > 0, tf.float32) * tf.cast(
            intersection[1] > 0, tf.float32)

        inter_square = tf.multiply(mask, inter_square)

        #calculate the boxs1 square and boxs2 square
        square1 = tf.multiply((boxes1[2] - boxes1[0]), (boxes1[3] - boxes1[1]))
        square2 = tf.multiply((boxes2[2] - boxes2[0]), (boxes2[3] - boxes2[1]))

        return inter_square / (square1 + square2 - inter_square +
                               1e-6), inter_square
Example #10
0
def sg_summary_activation(tensor, prefix='30. activation'):
    # defaults
    prefix = '' if prefix is None else prefix + '/'
    # summary name
    name = prefix + _pretty_name(tensor)
    # summary statistics
    with tf.name_scope('summary'):
        tf.scalar_summary(name + '/norm', tf.global_norm([tensor]))
        tf.scalar_summary(
            name + '/ratio',
            tf.reduce_mean(tf.cast(tf.greater(tensor, 0), tf.sg_floatx)))
        tf.histogram_summary(name, tensor)
Example #11
0
def sg_int(tensor, opt):
    r"""Casts a tensor to intx.
    
    See `tf.cast()` in tensorflow.

    Args:
      tensor: A `Tensor` or `SparseTensor` (automatically given by chain).
      opt:
        name: If provided, it replaces current tensor's name.

    Returns:
      A `Tensor` or `SparseTensor` with same shape as `tensor`.
    """
    return tf.cast(tensor, tf.sg_intx, name=opt.name)
Example #12
0
def rnn_classify(x, num_classes, is_test=False):
    with tf.sg_context(name='rnn_classify'):
        fw_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True)
        bw_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True)

        words_used_in_sent = tf.sign(tf.reduce_max(tf.abs(x), reduction_indices=2))
        length = tf.cast(tf.reduce_sum(words_used_in_sent, reduction_indices=1), tf.int32)

        outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, x, dtype=tf.float32, sequence_length=length)
        output = tf.concat(outputs, 2).sg_reshape(shape=[-1, 2 * latent_dim])

        prediction = output.sg_dense(dim=num_classes, name='dense')
        res = tf.reshape(prediction, [x.get_shape().as_list()[0], -1, num_classes])

    return res
Example #13
0
def sg_cast(tensor, opt):
    r"""Casts a tensor to a new type.
    
    See `tf.cast()` in tensorflow.

    Args:
      tensor: A `Tensor` or `SparseTensor` (automatically given by chain).
      opt:
        dtype : The destination type.
        name : If provided, it replaces current tensor's name

    Returns:
      A `Tensor` or `SparseTensor` with same shape as `tensor`.
    """
    assert opt.dtype is not None, 'dtype is mandatory.'
    return tf.cast(tensor, opt.dtype, name=opt.name)
Example #14
0
def sg_hinge(tensor, opt):
    assert opt.target is not None, 'target is mandatory.'

    # default margin
    opt += tf.sg_opt(margin=1)

    # reshape target
    shape = tensor.get_shape().as_list()
    broadcast_shape = [-1] + [1] * (len(shape) - 2) + [shape[-1]]
    target = tf.cast(tf.reshape(opt.target, broadcast_shape), tf.sg_floatx)

    # hinge loss
    out = tf.identity(tf.maximum(opt.margin - target * tensor, 0), 'hinge')

    # add summary
    tf.sg_summary_loss(out)

    return out
Example #15
0
def sg_summary_activation(tensor, prefix=None, name=None):
    r"""Register `tensor` to summary report as `activation`

    Args:
      tensor: A `Tensor` to log as activation
      prefix: A `string`. A prefix to display in the tensor board web UI.
      name: A `string`. A name to display in the tensor board web UI.

    Returns:
      None
    """
    # defaults
    prefix = '' if prefix is None else prefix + '/'
    # summary name
    name = prefix + _pretty_name(tensor) if name is None else prefix + name
    # summary statistics
    _scalar(name + '/ratio',
            tf.reduce_mean(tf.cast(tf.greater(tensor, 0), tf.sg_floatx)))
    _histogram(name + '/ratio-h', tensor)
Example #16
0
def sg_hinge(tensor, opt):
    r"""Returns hinge loss between `tensor` and `target`.
    
    Args:
      tensor: A `Tensor`.
      opt:
        target: A `Tensor`. Labels.
        margin: An int. Maximum margin. Default is 1.
        name: A `string`. A name to display in the tensor board web UI.
      
    Returns:
      A `Tensor`.
    
    For example,
    
    ```
    tensor = [[30, 10, 40], [13, 30, 42]]
    target = [[0, 0, 1], [0, 1, 0]]
    tensor.sg_hinge(target=target, one_hot=True) =>     [[ 1.  1.  0.]
                                                         [ 1.  0.  1.]]
    ```
    """
    assert opt.target is not None, 'target is mandatory.'

    # default margin
    opt += tf.sg_opt(margin=1)

    # reshape target
    shape = tensor.get_shape().as_list()
    broadcast_shape = [-1] + [1] * (len(shape) - 2) + [shape[-1]]
    target = tf.cast(tf.reshape(opt.target, broadcast_shape), tf.sg_floatx)

    # hinge loss
    out = tf.identity(tf.maximum(opt.margin - target * tensor, 0), 'hinge')

    # add summary
    tf.sg_summary_loss(out, name=opt.name)

    return out
def tower_infer_enc(chars,
                    scope,
                    rnn_cell,
                    dec_cell,
                    word_emb,
                    out_reuse_vars=False,
                    dev='/cpu:0'):
    out_rvars = out_reuse_vars

    # make embedding matrix for source and target
    with tf.device(dev):
        with tf.variable_scope('embatch_size', reuse=out_reuse_vars):
            # (vocab_size, latent_dim)
            emb_char = tf.sg_emb(name='emb_char',
                                 voca_size=Hp.char_vs,
                                 dim=Hp.hd,
                                 dev=dev)
            emb_word = tf.sg_emb(name='emb_word',
                                 emb=word_emb,
                                 voca_size=Hp.word_vs,
                                 dim=300,
                                 dev=dev)

    chars = tf.cast(chars, tf.int32)

    time = tf.constant(0)

    inputs = tf.transpose(chars, perm=[1, 0, 2])
    input_ta = tensor_array_ops.TensorArray(tf.int32,
                                            size=tf.shape(chars)[1],
                                            dynamic_size=True,
                                            clear_after_read=True)
    chars_sent = input_ta.unstack(inputs)  #each element is (batch, sentlen)

    resp_steps = tf.shape(chars)[1]  # number of sentences in paragraph
    statm_steps = resp_steps // 2

    rnn_state = rnn_cell.zero_state(
        Hp.batch_size, tf.float32)  #rnn_cell.rnn_state, rnn_cell.rnn_h
    maxdecode = 3

    # -------------------------------------------- STATEMENT ENCODING -----------------------------------------------

    def rnn_cond_stat(time, rnn_state):
        return tf.less(time, statm_steps - 1)

    def rnn_body_stat(time, rnn_state):
        ch = chars_sent.read(time)
        ch = tf.reverse_sequence(input=ch,
                                 seq_lengths=[Hp.c_maxlen] * Hp.batch_size,
                                 seq_dim=1)
        reuse_vars = out_reuse_vars

        # --------------------------   BYTENET ENCODER   --------------------------

        with tf.variable_scope('encoder'):
            # embed table lookup
            enc = ch.sg_lookup(emb=emb_char)  #(batch, sentlen, latentdim)
            # loop dilated conv block
            for i in range(Hp.num_blocks):
                enc = (enc.sg_res_block(size=5,
                                        rate=1,
                                        name="enc1_%d" % (i),
                                        is_first=True,
                                        reuse_vars=reuse_vars,
                                        dev=dev).sg_res_block(
                                            size=5,
                                            rate=2,
                                            name="enc2_%d" % (i),
                                            reuse_vars=reuse_vars,
                                            dev=dev).sg_res_block(
                                                size=5,
                                                rate=4,
                                                name="enc4_%d" % (i),
                                                reuse_vars=reuse_vars,
                                                dev=dev).sg_res_block(
                                                    size=5,
                                                    rate=8,
                                                    name="enc8_%d" % (i),
                                                    reuse_vars=reuse_vars,
                                                    dev=dev).sg_res_block(
                                                        size=5,
                                                        rate=16,
                                                        name="enc16_%d" % (i),
                                                        reuse_vars=reuse_vars,
                                                        dev=dev))
            byte_enc = enc
            # --------------------------   QCNN + QPOOL ENCODER #1  --------------------------

            with tf.variable_scope('quazi'):

                #quasi cnn layer ZFO  [batch * 3, seqlen, dim2 ]
                conv = byte_enc.sg_quasi_conv1d(is_enc=True,
                                                size=4,
                                                name="qconv_1",
                                                dev=dev,
                                                reuse_vars=reuse_vars)
                # c = f * c + (1 - f) * z, h = o*c [batch * 4, seqlen, hd]
                pool0 = conv.sg_quasi_rnn(is_enc=False,
                                          att=False,
                                          name="qrnn_1",
                                          reuse_vars=reuse_vars,
                                          dev=dev)

                qpool_last = pool0[:, -1, :]

        # --------------------------   MAXPOOL along time dimension   --------------------------

        inpt_maxpl = tf.expand_dims(byte_enc,
                                    1)  # [batch, 1, seqlen, channels]
        maxpool = tf.nn.max_pool(inpt_maxpl, [1, 1, Hp.c_maxlen, 1],
                                 [1, 1, 1, 1], 'VALID')
        maxpool = tf.squeeze(maxpool, [1, 2])

        # --------------------------   HIGHWAY   --------------------------

        concat = qpool_last + maxpool
        with tf.variable_scope('highway', reuse=reuse_vars):
            input_lstm = highway(concat, concat.get_shape()[-1], num_layers=1)

        # --------------------------   CONTEXT LSTM  --------------------------
        input_lstm = tf.nn.dropout(input_lstm, Hp.keep_prob)

        with tf.variable_scope('contx_lstm', reuse=reuse_vars):
            output, rnn_state = rnn_cell(input_lstm, rnn_state)

        return (time + 1, rnn_state)

    loop_vars_stat = [time, rnn_state]

    time, rnn_state = tf.while_loop\
                      (rnn_cond_stat, rnn_body_stat, loop_vars_stat, swap_memory=False)

    return rnn_state
Example #18
0
    def train(self):
        ''' Network '''
        batch_pred_feats, batch_pred_coords, batch_pred_confs, self.final_state = self.LSTM(
            'lstm', self.x)

        iou_predict_truth, intersection = self.iou(batch_pred_coords,
                                                   self.y[:, 0:4])

        should_exist = I = tf.cast(
            tf.reduce_sum(self.y[:, 0:4], axis=1) > 0., tf.float32)
        no_I = tf.ones_like(I, dtype=tf.float32) - I

        object_loss = tf.nn.l2_loss(
            I * (batch_pred_confs - iou_predict_truth)) * self.object_scale
        noobject_loss = tf.nn.l2_loss(
            no_I *
            (batch_pred_confs - iou_predict_truth)) * self.noobject_scale

        p_sqrt_w = tf.sqrt(
            tf.minimum(1.0, tf.maximum(0.0, batch_pred_coords[:, 2])))
        p_sqrt_h = tf.sqrt(
            tf.minimum(1.0, tf.maximum(0.0, batch_pred_coords[:, 3])))

        sqrt_w = tf.sqrt(tf.abs(self.y[:, 2]))
        sqrt_h = tf.sqrt(tf.abs(self.y[:, 3]))

        loss = (tf.nn.l2_loss(I * (batch_pred_coords[:, 0] - self.y[:, 0])) +
                tf.nn.l2_loss(I * (batch_pred_coords[:, 1] - self.y[:, 1])) +
                tf.nn.l2_loss(I * (p_sqrt_w - sqrt_w)) +
                tf.nn.l2_loss(I * (p_sqrt_h - sqrt_h))) * self.coord_scale

        #max_iou = tf.nn.l2_loss(I*(tf.ones_like(iou_predict_truth, dtype=tf.float32) - iou_predict_truth))

        total_loss = loss + object_loss + noobject_loss  #+ max_iou
        ''' Optimizer '''
        optimizer = tf.train.AdamOptimizer(
            learning_rate=self.learning_rate).minimize(
                total_loss)  # Adam Optimizer
        ''' Summary for tensorboard analysis '''
        dataset_loss = -1
        dataset_loss_best = 100
        test_writer = tf.summary.FileWriter('summary/test')
        tf.summary.scalar('dataset_loss', dataset_loss)
        summary_op = tf.summary.merge_all()
        ''' Initializing the variables '''
        self.saver = tf.train.Saver()
        batch_states = np.zeros((self.batchsize, 2 * self.len_vec))

        # TODO: make this a command line argument, etc.
        # training set loader
        batch_loader = BatchLoader(
            "./DATA/TRAINING/",
            seq_len=self.nsteps,
            batch_size=self.batchsize,
            step_size=1,
            folders_to_use=[
                "GOPR0005", "GOPR0006", "GOPR0008", "GOPR0008_2", "GOPR0009",
                "GOPR0009_2", "GOPR0010", "GOPR0011", "GOPR0012", "GOPR0013",
                "GOPR0014", "GOPR0015", "GOPR0016", "MVI_8607", "MVI_8609",
                "MVI_8610", "MVI_8612", "MVI_8614", "MVI_8615", "MVI_8616"
            ])
        validation_set_loader = BatchLoader(
            "./DATA/VALID/",
            seq_len=self.nsteps,
            batch_size=self.batchsize,
            step_size=1,
            folders_to_use=[
                "bbd_2017__2017-01-09-21-40-02_cam_flimage_raw",
                "bbd_2017__2017-01-09-21-44-31_cam_flimage_raw",
                "bbd_2017__2017-01-09-21-48-46_cam_flimage_raw",
                "bbd_2017__2017-01-10-16-07-49_cam_flimage_raw",
                "bbd_2017__2017-01-10-16-21-01_cam_flimage_raw",
                "bbd_2017__2017-01-10-16-31-57_cam_flimage_raw",
                "bbd_2017__2017-01-10-21-43-03_cam_flimage_raw",
                "bbd_2017__2017-01-11-20-21-32_cam_flimage_raw",
                "bbd_2017__2017-01-11-21-02-37_cam_flimage_raw"
            ])

        print("%d available training batches" % len(batch_loader.batches))
        print("%d available validation batches" %
              len(validation_set_loader.batches))
        ''' Launch the graph '''
        with tf.Session() as sess:
            if self.restore_weights == True and os.path.isfile(
                    self.rolo_current_save + ".index"):
                # sess.run(init)
                tf.sg_init(sess)
                self.saver.restore(sess, self.rolo_current_save)
                print("Weight loaded, finetuning")
            else:
                # sess.run(init)
                tf.sg_init(sess)
                print("Training from scratch")

            epoch_loss = []
            for self.iter_id in range(self.n_iters):
                ''' Load training data & ground truth '''
                batch_id = self.iter_id - self.batch_offset

                batch_xs, batch_ys, _ = batch_loader.load_batch(batch_id)
                ''' Update weights by back-propagation '''

                sess.run(optimizer,
                         feed_dict={
                             self.x: batch_xs,
                             self.y: batch_ys
                         })

                if self.iter_id % self.display_step == 0:
                    ''' Calculate batch loss '''
                    batch_loss = sess.run(total_loss,
                                          feed_dict={
                                              self.x: batch_xs,
                                              self.y: batch_ys
                                          })
                    epoch_loss.append(batch_loss)
                    print("Total Batch loss for iteration %d: %.9f" %
                          (self.iter_id, batch_loss))

                if self.iter_id % self.display_step == 0:
                    ''' Calculate batch loss '''
                    batch_loss = sess.run(loss,
                                          feed_dict={
                                              self.x: batch_xs,
                                              self.y: batch_ys
                                          })
                    print(
                        "Bounding box coord error loss for iteration %d: %.9f"
                        % (self.iter_id, batch_loss))

                if self.display_object_loss and self.iter_id % self.display_step == 0:
                    ''' Calculate batch object loss '''
                    batch_o_loss = sess.run(object_loss,
                                            feed_dict={
                                                self.x: batch_xs,
                                                self.y: batch_ys
                                            })
                    print("Object loss for iteration %d: %.9f" %
                          (self.iter_id, batch_o_loss))

                if self.display_object_loss and self.iter_id % self.display_step == 0:
                    ''' Calculate batch object loss '''
                    batch_noo_loss = sess.run(noobject_loss,
                                              feed_dict={
                                                  self.x: batch_xs,
                                                  self.y: batch_ys
                                              })
                    print("No Object loss for iteration %d: %.9f" %
                          (self.iter_id, batch_noo_loss))

                if self.iou_with_ground_truth and self.iter_id % self.display_step == 0:
                    ''' Calculate batch object loss '''
                    batch_o_loss = sess.run(tf.reduce_mean(iou_predict_truth),
                                            feed_dict={
                                                self.x: batch_xs,
                                                self.y: batch_ys
                                            })
                    print("Average IOU with ground for iteration %d: %.9f" %
                          (self.iter_id, batch_o_loss))

                if self.display_coords is True and self.iter_id % self.display_step == 0:
                    ''' Caculate predicted coordinates '''
                    coords_predict = sess.run(batch_pred_coords,
                                              feed_dict={
                                                  self.x: batch_xs,
                                                  self.y: batch_ys
                                              })
                    print("predicted coords:" + str(coords_predict[0]))
                    print("ground truth coords:" + str(batch_ys[0]))
                ''' Save model '''
                if self.iter_id % self.save_step == 1:
                    self.saver.save(sess, self.rolo_current_save)
                    print("\n Model saved in file: %s" %
                          self.rolo_current_save)
                ''' Validation '''
                if self.validate == True and self.iter_id % self.validate_step == 0 and self.iter_id > 0:
                    # Run validation set

                    dataset_loss = self.test(sess, total_loss,
                                             validation_set_loader,
                                             batch_pred_feats,
                                             batch_pred_coords,
                                             batch_pred_confs,
                                             self.final_state)
                    ''' Early-stop regularization '''
                    if dataset_loss <= dataset_loss_best:
                        dataset_loss_best = dataset_loss
                        self.saver.save(sess, self.rolo_weights_file)
                        print("\n Better Model saved in file: %s" %
                              self.rolo_weights_file)
                    ''' Write summary for tensorboard '''
                    summary = sess.run(summary_op,
                                       feed_dict={
                                           self.x: batch_xs,
                                           self.y: batch_ys
                                       })
                    test_writer.add_summary(summary, self.iter_id)
            print("Average total loss %f" % np.mean(epoch_loss))
        return
Example #19
0
def sg_float(tensor, opt):
    return tf.cast(tensor, tf.sg_floatx)
Example #20
0
def sg_int(tensor, opt):
    return tf.cast(tensor, tf.sg_intx)
Example #21
0
def sg_int(tensor, opt):
    return tf.cast(tensor, tf.sg_intx, name=opt.name)
Example #22
0
def sg_float(tensor, opt):
    return tf.cast(tensor, tf.sg_floatx, name=opt.name)
Example #23
0
            gvs = optimizer.compute_gradients(cost)

            def ClipIfNotNone(grad):
                if grad is None:
                    return grad
                return tf.clip_by_value(grad, -1, 1)

            capped_gvs = [(ClipIfNotNone(grad), var) for grad, var in gvs]
            train_optimizer = optimizer.apply_gradients(capped_gvs)

        with tf.name_scope('decoder'):
            decoded, log_prob = tf.nn.ctc_greedy_decoder(logits, seq_len)

        with tf.name_scope('LER'):
            ler = tf.reduce_mean(
                tf.edit_distance(tf.cast(decoded[0], tf.int32), targets))
        tf.summary.scalar("LER", ler)

        merged = tf.summary.merge_all()


def train_loop():
    with tf.device("/cpu:0"):
        # Launch the graph
        with tf.Session(graph=graph, config=config) as sess:
            print("Starting Tensorboard...")
            initstart = time.time()
            train_writer = tf.summary.FileWriter(logs_path + '/TRAIN',
                                                 graph=sess.graph)
            test_writer = tf.summary.FileWriter(logs_path + '/TEST',
                                                graph=sess.graph)
Example #24
0
    def __init__(self,
                 x,
                 y,
                 num_batch,
                 vocab_size,
                 emb_dim,
                 hidden_dim,
                 max_ep=240,
                 infer_shape=(1, 1),
                 mode="train"):

        self.num_batch = num_batch
        self.emb_dim = emb_dim
        self.hidden_dim = hidden_dim
        self.vocab_size = vocab_size
        self.max_len_infer = 512
        self.max_ep = max_ep

        # reuse = len([t for t in tf.global_variables() if t.name.startswith('gen')]) > 0
        reuse = (mode == 'infer')

        if mode == "train":
            self.x = x
            self.y = y
        elif mode == "infer":
            self.x = tf.placeholder(tf.int32, shape=infer_shape)
            self.y = tf.placeholder(tf.int32, shape=infer_shape)

        with tf.variable_scope("gen_embs", reuse=reuse):
            self.emb_x = tf.get_variable("emb_x",
                                         [self.vocab_size, self.emb_dim])
            self.emb_y = tf.get_variable("emb_y",
                                         [self.vocab_size, self.emb_dim])
            self.X = tf.nn.embedding_lookup(self.emb_x, self.x)
            self.Y = tf.nn.embedding_lookup(self.emb_y, self.y)

        with tf.sg_context(name='gen', reuse=reuse):
            #     self.emb_x = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_x")
            #     self.emb_y = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_y")
            # self.emb_x = tf.sg_emb(name='emb_x', voca_size=self.vocab_size, dim=self.emb_dim)  # (68,16)
            # self.emb_y = tf.sg_emb(name='emb_y', voca_size=self.vocab_size, dim=self.emb_dim)  # (68,16)
            # self.X = self.x.sg_lookup(emb=self.emb_x)  # (8,63,16)
            # self.Y = self.y.sg_lookup(emb=self.emb_y)  # (8,63,16)

            if mode == "train":
                self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
                                                 dim=self.vocab_size,
                                                 name="lstm")  # (8, 63, 68)
                self.test = self.lstm_layer.sg_softmax(name="testtt")

                print "mazum??"
                print self.test

            elif mode == "infer":
                self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
                                                 dim=self.vocab_size,
                                                 last_only=True,
                                                 name="lstm")
                self.log_prob = tf.log(self.lstm_layer)

                # next_token: select by distribution probability, preds: select by argmax

                self.multinormed = tf.multinomial(self.log_prob, 1)
                self.next_token = tf.cast(
                    tf.reshape(tf.multinomial(self.log_prob, 1),
                               [1, infer_shape[0]]), tf.int32)
                self.preds = self.lstm_layer.sg_argmax()

        if mode == "train":
            self.loss = self.lstm_layer.sg_ce(target=self.y)
            self.istarget = tf.not_equal(self.y, 0).sg_float()

            self.reduced_loss = (self.loss.sg_sum()) / (
                self.istarget.sg_sum() + 0.0000001)
            tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
Example #25
0
File: nn.py Project: jackyzha0/vybe
W = tf.Variable(
    tf.random_normal([n_hidden_units_three, num_classes], mean=0, stddev=sd))
b = tf.Variable(tf.random_normal([num_classes], mean=0, stddev=sd))
with tf.name_scope('out'):
    y_ = tf.nn.softmax(tf.matmul(h_3, W) + b, name="out")

init = tf.global_variables_initializer()

cost_function = tf.reduce_mean(
    -tf.reduce_sum(Y * tf.log(y_), reduction_indices=[1]))
#optimizer = tf.train.RMSPropOptimizer(learning_rate,decay=0.9,momentum=0.9,centered=True).minimize(cost_function)
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(
    cost_function)

correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

cost_history = np.empty(shape=[1], dtype=float)
acc_history = np.empty(shape=[1], dtype=float)
t_cost_history = np.empty(shape=[1], dtype=float)
t_acc_history = np.empty(shape=[1], dtype=float)
y_true, y_pred = None, None

with tf.Session() as session:
    session.run(init)
    saver = tf.train.Saver()
    for epoch in range(epochs):
        for batch in range(int(db_size / batchsize)):
            indices = get_indices(batchsize)
            feed = data_tools.next_minibatch(indices, db)
            print("Batch", batch, "in epoch", epoch)
Example #26
0
def sg_cast(tensor, opt):
    assert opt.dtype is not None, 'dtype is mandatory.'
    return tf.cast(tensor, opt.dtype, name=opt.name)
Example #27
0
def tower_loss_manyparams(xx, scope, reu_vars=False):
    # make embedding matrix for source and target
    reu_vars = reu_vars
    with tf.variable_scope('embatch_size', reuse=reu_vars):
        # (vocab_size, latent_dim)
        emb_x = tf.sg_emb(name='emb_x',
                          voca_size=Hp.vs,
                          dim=Hp.hd,
                          dev=self._dev)
        emb_y = tf.sg_emb(name='emb_y',
                          voca_size=Hp.vs,
                          dim=Hp.hd,
                          dev=self._dev)

    xx = tf.cast(xx, tf.int32)

    time = tf.constant(0)
    losses_int = tf.constant(0.0)
    inputs = tf.transpose(xx, perm=[1, 0, 2])
    input_ta = tensor_array_ops.TensorArray(tf.int32,
                                            size=1,
                                            dynamic_size=True,
                                            clear_after_read=False)
    x_sent = input_ta.unstack(inputs)  #each element is (batch, sentlen)

    n_steps = tf.shape(xx)[1]  # number of sentences in paragraph

    # generate first an unconditioned sentence
    n_input = Hp.hd
    subrec1_init = subrec_zero_state(Hp.batch_size, Hp.hd)
    subrec2_init = subrec_zero_state(Hp.batch_size, Hp.hd)

    with tf.variable_scope("mem", reuse=reu_vars) as scp:
        rnn_cell = LSTMCell(in_dim=h, dim=Hp.hd)
        crnn_cell = ConvLSTMCell(seqlen=Hp.maxlen,
                                 in_dim=n_input // 2,
                                 dim=Hp.hd // 2)

    (rnn_state_init, rnn_h_init) = rnn_cell.zero_state(Hp.batch_size)

    #   (batch, sentlen, latentdim/2)
    (crnn_state_init, crnn_h_init) = crnn_cell.zero_state(Hp.batch_size)

    def rnn_cond(time, subrec1, subrec2, rnn_state, rnn_h, crnn_state, crnn_h,
                 losses):
        return tf.less(time, n_steps - 1)

    def rnn_body(time, subrec1, subrec2, rnn_state, rnn_h, crnn_state, crnn_h,
                 losses):
        x = x_sent.read(time)
        y = x_sent.read(time + 1)  #   (batch, sentlen) = (16, 200)

        # shift target by one step for training source
        y_src = tf.concat([tf.zeros((Hp.batch_size, 1), tf.int32), y[:, :-1]],
                          1)
        reuse_vars = time == tf.constant(0) or reu_vars

        # --------------------------   BYTENET ENCODER   --------------------------

        # embed table lookup
        enc = x.sg_lookup(emb=emb_x)  #(batch, sentlen, latentdim)
        # loop dilated conv block
        for i in range(num_blocks):
            enc = (enc.sg_res_block(
                size=5, rate=1, name="enc1_%d" % (i),
                reuse_vars=reuse_vars).sg_res_block(
                    size=5,
                    rate=2,
                    name="enc2_%d" % (i),
                    reuse_vars=reuse_vars).sg_res_block(
                        size=5,
                        rate=4,
                        name="enc4_%d" % (i),
                        reuse_vars=reuse_vars).sg_res_block(
                            size=5,
                            rate=8,
                            name="enc8_%d" % (i),
                            reuse_vars=reuse_vars).sg_res_block(
                                size=5,
                                rate=16,
                                name="enc16_%d" % (i),
                                reuse_vars=reuse_vars))


# --------------------------   QCNN + QPOOL ENCODER with attention #1  --------------------------

#quasi cnn layer ZFO  [batch * 3, t, dim2 ]
        conv = enc.sg_quasi_conv1d(is_enc=True,
                                   size=3,
                                   name="qconv_1",
                                   reuse_vars=reuse_vars)
        #attention layer
        # recurrent layer # 1 + final encoder hidden state
        subrec1 = tf.tile((subrec1.sg_expand_dims(axis=1)), [1, Hp.maxlen, 1])
        concat = conv.sg_concat(target=subrec1,
                                axis=0)  # (batch*4, sentlen, latentdim)
        pool = concat.sg_quasi_rnn(is_enc=True,
                                   att=True,
                                   name="qrnn_1",
                                   reuse_vars=reuse_vars)
        subrec1 = pool[:Hp.batch_size, -1, :]  # last character in sequence

        # --------------------------   QCNN + QPOOL ENCODER with attention #2  --------------------------

        # quazi cnn ZFO (batch*3, sentlen, latentdim)
        conv = pool.sg_quasi_conv1d(is_enc=True,
                                    size=2,
                                    name="qconv_2",
                                    reuse_vars=reuse_vars)
        # (batch, sentlen-duplicated, latentdim)
        subrec2 = tf.tile((subrec2.sg_expand_dims(axis=1)), [1, Hp.maxlen, 1])
        # (batch*4, sentlen, latentdim)
        concat = conv.sg_concat(target=subrec2, axis=0)
        pool = concat.sg_quasi_rnn(is_enc=True,
                                   att=True,
                                   name="qrnn_2",
                                   reuse_vars=reuse_vars)
        subrec2 = pool[:Hp.batch_size, -1, :]  # last character in sequence

        # --------------------------   ConvLSTM with RESIDUAL connection and MULTIPLICATIVE block   --------------------------

        #residual block
        causal = False  # for encoder
        crnn_input = (pool[:Hp.batch_size, :, :].sg_bypass_gpus(
            name='relu_0', act='relu', bn=(not causal),
            ln=causal).sg_conv1d_gpus(name="dimred_0",
                                      size=1,
                                      dev="/cpu:0",
                                      reuse=reuse_vars,
                                      dim=Hp.hd / 2,
                                      act='relu',
                                      bn=(not causal),
                                      ln=causal))

        # conv LSTM
        with tf.variable_scope("mem/clstm") as scp:
            (crnn_state, crnn_h) = crnn_cell(crnn_input, (crnn_state, crnn_h),
                                             size=5,
                                             reuse_vars=reuse_vars)
        # dimension recover and residual connection
        rnn_input0 = pool[:Hp.batch_size,:,:] + crnn_h\
                    .sg_conv1d_gpus(name = "diminc_0",size=1,dev="/cpu:0", dim=Hp.hd,reuse=reuse_vars, act='relu', bn=(not causal), ln=causal)

        # --------------------------   QCNN + QPOOL ENCODER with attention #3  --------------------------

        # pooling for lstm input
        # quazi cnn ZFO (batch*3, sentlen, latentdim)
        conv = rnn_input0.sg_quasi_conv1d(is_enc=True,
                                          size=2,
                                          name="qconv_3",
                                          reuse_vars=reuse_vars)
        pool = conv.sg_quasi_rnn(is_enc=True,
                                 att=False,
                                 name="qrnn_3",
                                 reuse_vars=reuse_vars)
        rnn_input = pool[:Hp.batch_size, -1, :]  # last character in sequence

        # --------------------------   LSTM with RESIDUAL connection and MULTIPLICATIVE block --------------------------

        # recurrent block
        with tf.variable_scope("mem/lstm") as scp:
            (rnn_state, rnn_h) = rnn_cell(rnn_input, (rnn_state, rnn_h))

        rnn_h2 = tf.tile(((rnn_h + rnn_input).sg_expand_dims(axis=1)),
                         [1, Hp.maxlen, 1])

        # --------------------------   BYTENET DECODER   --------------------------

        # CNN decoder
        dec = y_src.sg_lookup(emb=emb_y).sg_concat(target=rnn_h2, name="dec")

        for i in range(num_blocks):
            dec = (dec.sg_res_block(
                size=3,
                rate=1,
                causal=True,
                name="dec1_%d" % (i),
                reuse_vars=reuse_vars).sg_res_block(
                    size=3,
                    rate=2,
                    causal=True,
                    name="dec2_%d" % (i),
                    reuse_vars=reuse_vars).sg_res_block(
                        size=3,
                        rate=4,
                        causal=True,
                        name="dec4_%d" % (i),
                        reuse_vars=reuse_vars).sg_res_block(
                            size=3,
                            rate=8,
                            causal=True,
                            name="dec8_%d" % (i),
                            reuse_vars=reuse_vars).sg_res_block(
                                size=3,
                                rate=16,
                                causal=True,
                                name="dec16_%d" % (i),
                                reuse_vars=reuse_vars))

        # final fully convolution layer for softmax
        dec = dec.sg_conv1d_gpus(size=1,
                                 dim=Hp.vs,
                                 name="out",
                                 summary=False,
                                 dev=self._dev,
                                 reuse=reuse_vars)

        ce_array = dec.sg_ce(target=y, mask=True, name="cross_ent_example")
        cross_entropy_mean = tf.reduce_mean(ce_array, name='cross_entropy')

        losses = tf.add_n([losses, cross_entropy_mean], name='total_loss')

        return (time + 1, subrec1, subrec2, rnn_state, rnn_h, crnn_state,
                crnn_h, losses)