Пример #1
0
    def __init__(self, config, num_words, device='', reuse=False):
        AbstractNetwork.__init__(self, "oracle", device=device)

        with tf.variable_scope(self.scope_name, reuse=reuse):
            embeddings = []
            self.batch_size = None

            # QUESTION
            self._is_training = tf.placeholder(tf.bool, name="is_training")
            self._question = tf.placeholder(tf.int32, [self.batch_size, None],
                                            name='question')
            self._seq_length = tf.placeholder(tf.int32, [self.batch_size],
                                              name='seq_length')

            word_emb = utils.get_embedding(
                self._question,
                n_words=num_words,
                n_dim=int(config['model']['question']["embedding_dim"]),
                scope="word_embedding")

            lstm_states, _ = rnn.variable_length_LSTM(
                word_emb,
                num_hidden=int(config['model']['question']["no_LSTM_hiddens"]),
                seq_length=self._seq_length)
            embeddings.append(lstm_states)

            # CATEGORY
            if config['inputs']['category']:
                self._category = tf.placeholder(tf.int32, [self.batch_size],
                                                name='category')

                cat_emb = utils.get_embedding(
                    self._category,
                    int(config['model']['category']["n_categories"]) +
                    1,  # we add the unkwon category
                    int(config['model']['category']["embedding_dim"]),
                    scope="cat_embedding")
                embeddings.append(cat_emb)
                print("Input: Category")

            # SPATIAL
            if config['inputs']['spatial']:
                self._spatial = tf.placeholder(tf.float32,
                                               [self.batch_size, 8],
                                               name='spatial')
                embeddings.append(self._spatial)
                print("Input: Spatial")

            # IMAGE
            if config['inputs']['image']:
                self._image = tf.placeholder(tf.float32, [self.batch_size] +
                                             config['model']['image']["dim"],
                                             name='image')

                if len(config['model']["image"]["dim"]) == 1:
                    self.image_out = self._image
                else:
                    self.image_out = attention.attention_factory(
                        self._image, lstm_states,
                        config['model']["image"]["attention"])

                embeddings.append(self.image_out)
                print("Input: Image")

            # CROP
            if config['inputs']['crop']:
                self._crop = tf.placeholder(tf.float32, [self.batch_size] +
                                            config['model']['crop']["dim"],
                                            name='crop')

                if len(config['model']["crop"]["dim"]) == 1:
                    self.crop_out = self._crop
                else:
                    self.crop_out = attention.attention_factory(
                        self._crop, lstm_states,
                        config['model']['crop']["attention"])

                embeddings.append(self.crop_out)
                print("Input: Crop")

            # Compute the final embedding
            emb = tf.concat(embeddings, axis=1)

            # OUTPUT
            num_classes = 3
            self._answer = tf.placeholder(tf.float32,
                                          [self.batch_size, num_classes],
                                          name='answer')

            with tf.variable_scope('mlp'):
                num_hiddens = config['model']['MLP']['num_hiddens']
                l1 = utils.fully_connected(emb,
                                           num_hiddens,
                                           activation='relu',
                                           scope='l1')

                self.pred = utils.fully_connected(l1,
                                                  num_classes,
                                                  activation='softmax',
                                                  scope='softmax')
                self.best_pred = tf.argmax(self.pred, axis=1)

            self.loss = tf.reduce_mean(
                utils.cross_entropy(self.pred, self._answer))
            self.error = tf.reduce_mean(utils.error(self.pred, self._answer))

            print('Model... Oracle build!')
Пример #2
0
    def __init__(self, config, num_words, policy_gradient, device='', reuse=False):
        AbstractNetwork.__init__(self, "qgen", device=device)

        # Create the scope for this graph
        with tf.variable_scope(self.scope_name, reuse=reuse):

            mini_batch_size = None

            # Picture
            self.images = tf.placeholder(tf.float32, [mini_batch_size] + config['image']["dim"], name='images')

            # Question
            self.dialogues = tf.placeholder(tf.int32, [mini_batch_size, None], name='dialogues')
            self.answer_mask = tf.placeholder(tf.float32, [mini_batch_size, None], name='answer_mask')  # 1 if keep and (1 q/a 1) for (START q/a STOP)
            self.padding_mask = tf.placeholder(tf.float32, [mini_batch_size, None], name='padding_mask')
            self.seq_length = tf.placeholder(tf.int32, [mini_batch_size], name='seq_length')

            # Rewards
            self.cum_rewards = tf.placeholder(tf.float32, shape=[mini_batch_size, None], name='cum_reward')

            # DECODER Hidden state (for beam search)
            zero_state = tf.zeros([1, config['num_lstm_units']])  # default LSTM state is a zero-vector
            zero_state = tf.tile(zero_state, [tf.shape(self.images)[0], 1])  # trick to do a dynamic size 0 tensors

            self.decoder_zero_state_c = tf.placeholder_with_default(zero_state, [mini_batch_size, config['num_lstm_units']], name="state_c")
            self.decoder_zero_state_h = tf.placeholder_with_default(zero_state, [mini_batch_size, config['num_lstm_units']], name="state_h")
            decoder_initial_state = tf.contrib.rnn.LSTMStateTuple(c=self.decoder_zero_state_c, h=self.decoder_zero_state_h)

            # Misc
            self.is_training = tf.placeholder(tf.bool, name='is_training')
            self.greedy = tf.placeholder_with_default(False, shape=(), name="greedy") # use for graph
            self.samples = None

            # remove last token
            input_dialogues = self.dialogues[:, :-1]
            input_seq_length = self.seq_length - 1

            # remove first token(=start token)
            rewards = self.cum_rewards[:, 1:]
            target_words = self.dialogues[:, 1:]

            # to understand the padding:
            # input
            #   <start>  is   it   a    blue   <?>   <yes>   is   it  a    car  <?>   <no>   <stop_dialogue>
            # target
            #    is      it   a   blue   <?>    -      is    it   a   car  <?>   -   <stop_dialogue>  -



            # image processing
            if len(config["image"]["dim"]) == 1:
                self.image_out = self.images
            else:
                self.image_out = get_attention(self.images, None, "none") #TODO: improve by using the previous lstm state?


            # Reduce the embedding size of the image
            with tf.variable_scope('picture_embedding'):
                self.picture_emb = utils.fully_connected(self.image_out,
                                                    config['picture_embedding_size'])
                picture_emb = tf.expand_dims(self.picture_emb, 1)
                picture_emb = tf.tile(picture_emb, [1, tf.shape(input_dialogues)[1], 1])

            # Compute the question embedding
            input_words = utils.get_embedding(
                input_dialogues,
                n_words=num_words,
                n_dim=config['word_embedding_size'],
                scope="word_embedding")

            # concat word embedding and picture embedding
            decoder_input = tf.concat([input_words, picture_emb], axis=2, name="concat_full_embedding")


            # encode one word+picture
            decoder_lstm_cell = tf.contrib.rnn.LayerNormBasicLSTMCell(
                    config['num_lstm_units'],
                    layer_norm=False,
                    dropout_keep_prob=1.0,
                    reuse=reuse)


            self.decoder_output, self.decoder_state = tf.nn.dynamic_rnn(
                cell=decoder_lstm_cell,
                inputs=decoder_input,
                dtype=tf.float32,
                initial_state=decoder_initial_state,
                sequence_length=input_seq_length,
                scope="word_decoder")  # TODO: use multi-layer RNN

            max_sequence = tf.reduce_max(self.seq_length)

            # compute the softmax for evaluation
            with tf.variable_scope('decoder_output'):
                flat_decoder_output = tf.reshape(self.decoder_output, [-1, decoder_lstm_cell.output_size])
                flat_mlp_output = utils.fully_connected(flat_decoder_output, num_words)

                # retrieve the batch/dialogue format
                mlp_output = tf.reshape(flat_mlp_output, [tf.shape(self.seq_length)[0], max_sequence - 1, num_words])  # Ignore th STOP token

                self.softmax_output = tf.nn.softmax(mlp_output, name="softmax")
                self.argmax_output = tf.argmax(mlp_output, axis=2)

                self.cross_entropy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=mlp_output, labels=target_words)

            # compute the maximum likelihood loss
            with tf.variable_scope('ml_loss'):

                ml_loss = tf.identity(self.cross_entropy_loss)
                ml_loss *= self.answer_mask[:, 1:]  # remove answers (ignore the <stop> token)
                ml_loss *= self.padding_mask[:, 1:]  # remove padding (ignore the <start> token)

                # Count number of unmask elements
                count = tf.reduce_sum(self.padding_mask) - tf.reduce_sum(1 - self.answer_mask[:, :-1]) - 1  # no_unpad - no_qa - START token

                ml_loss = tf.reduce_sum(ml_loss, axis=1)  # reduce over dialogue dimension
                ml_loss = tf.reduce_sum(ml_loss, axis=0)  # reduce over minibatch dimension
                self.ml_loss = ml_loss / count  # Normalize

                self.loss = self.ml_loss

            # Compute policy gradient
            if policy_gradient:

                with tf.variable_scope('rl_baseline'):
                    decoder_out = tf.stop_gradient(self.decoder_output)  # take the LSTM output (and stop the gradient!)

                    flat_decoder_output = tf.reshape(decoder_out, [-1, decoder_lstm_cell.output_size])  #
                    flat_h1 = utils.fully_connected(flat_decoder_output, n_out=100, activation='relu', scope='baseline_hidden')
                    flat_baseline = utils.fully_connected(flat_h1, 1, activation='relu', scope='baseline_out')

                    self.baseline = tf.reshape(flat_baseline, [tf.shape(self.seq_length)[0], max_sequence-1])
                    self.baseline *= self.answer_mask[:, 1:]
                    self.baseline *= self.padding_mask[:, 1:]


                with tf.variable_scope('policy_gradient_loss'):

                    # Compute log_prob
                    self.log_of_policy = tf.identity(self.cross_entropy_loss)
                    self.log_of_policy *= self.answer_mask[:, 1:]  # remove answers (<=> predicted answer has maximum reward) (ignore the START token in the mask)
                    # No need to use padding mask as the discounted_reward is already zero once the episode terminated

                    # Policy gradient loss
                    rewards *= self.answer_mask[:, 1:]
                    self.score_function = tf.multiply(self.log_of_policy, rewards - self.baseline)  # score function

                    self.baseline_loss = tf.reduce_sum(tf.square(rewards - self.baseline))

                    self.policy_gradient_loss = tf.reduce_sum(self.score_function, axis=1)  # sum over the dialogue trajectory
                    self.policy_gradient_loss = tf.reduce_mean(self.policy_gradient_loss, axis=0)  # reduce over minibatch dimension

                    self.loss = self.policy_gradient_loss
Пример #3
0
    def __init__(self, config, num_words, device='', reuse=False):
        AbstractNetwork.__init__(self, "guesser", device=device)

        mini_batch_size = None

        with tf.variable_scope(self.scope_name, reuse=reuse):

            # Dialogues
            self.dialogues = tf.placeholder(tf.int32, [mini_batch_size, None],
                                            name='dialogues')
            self.seq_length = tf.placeholder(tf.int32, [mini_batch_size],
                                             name='seq_length')

            # Objects
            self.obj_mask = tf.placeholder(tf.float32, [mini_batch_size, None],
                                           name='obj_mask')
            self.obj_cats = tf.placeholder(tf.int32, [mini_batch_size, None],
                                           name='obj_cats')
            self.obj_spats = tf.placeholder(
                tf.float32, [mini_batch_size, None, config['spat_dim']],
                name='obj_spats')

            # Targets
            self.targets = tf.placeholder(tf.int32, [mini_batch_size],
                                          name="targets_index")

            self.object_cats_emb = utils.get_embedding(
                self.obj_cats,
                config['no_categories'] + 1,
                config['cat_emb_dim'],
                scope='cat_embedding')

            self.objects_input = tf.concat(
                [self.object_cats_emb, self.obj_spats], axis=2)
            self.flat_objects_inp = tf.reshape(
                self.objects_input,
                [-1, config['cat_emb_dim'] + config['spat_dim']])

            with tf.variable_scope('obj_mlp'):
                h1 = utils.fully_connected(self.flat_objects_inp,
                                           n_out=config['obj_mlp_units'],
                                           activation='relu',
                                           scope='l1')
                h2 = utils.fully_connected(h1,
                                           n_out=config['dialog_emb_dim'],
                                           activation='relu',
                                           scope='l2')

            obj_embs = tf.reshape(
                h2,
                [-1, tf.shape(self.obj_cats)[1], config['dialog_emb_dim']])

            # Compute the word embedding
            input_words = utils.get_embedding(self.dialogues,
                                              n_words=num_words,
                                              n_dim=config['word_emb_dim'],
                                              scope="input_word_embedding")

            last_states, _ = rnn.variable_length_LSTM(
                input_words,
                num_hidden=config['num_lstm_units'],
                seq_length=self.seq_length)

            last_states = tf.reshape(last_states,
                                     [-1, config['num_lstm_units'], 1])
            scores = tf.matmul(obj_embs, last_states)
            scores = tf.reshape(scores, [-1, tf.shape(self.obj_cats)[1]])

            def masked_softmax(scores, mask):
                # subtract max for stability
                scores = scores - tf.tile(
                    tf.reduce_max(scores, axis=(1, ), keep_dims=True),
                    [1, tf.shape(scores)[1]])
                # compute padded softmax
                exp_scores = tf.exp(scores)
                exp_scores *= mask
                exp_sum_scores = tf.reduce_sum(exp_scores,
                                               axis=1,
                                               keep_dims=True)
                return exp_scores / tf.tile(exp_sum_scores,
                                            [1, tf.shape(exp_scores)[1]])

            self.softmax = masked_softmax(scores, self.obj_mask)
            self.selected_object = tf.argmax(self.softmax, axis=1)

            self.loss = tf.reduce_mean(
                utils.cross_entropy(self.softmax, self.targets))
            self.error = tf.reduce_mean(utils.error(self.softmax,
                                                    self.targets))
Пример #4
0
    def __init__(self, config, num_words, device='', reuse=False):
        AbstractNetwork.__init__(self, "guesser", device=device)

        batch_size = None

        with tf.variable_scope(self.scope_name, reuse=reuse):

            self._is_training = tf.placeholder(tf.bool, name="is_training")

            dropout_keep_scalar = float(config["dropout_keep_prob"])
            dropout_keep = tf.cond(self._is_training,
                                   lambda: tf.constant(dropout_keep_scalar),
                                   lambda: tf.constant(1.0))

            #####################
            #   DIALOGUE
            #####################

            self._dialogue = tf.placeholder(tf.int32, [batch_size, None],
                                            name='dialogue')
            self._seq_length = tf.placeholder(tf.int32, [batch_size],
                                              name='seq_length_dialogue')

            word_emb = tfc_layers.embed_sequence(
                ids=self._dialogue,
                vocab_size=num_words,
                embed_dim=config["question"]["word_embedding_dim"],
                scope="word_embedding",
                reuse=reuse)

            if config["question"]['glove']:
                self._glove = tf.placeholder(tf.float32, [None, None, 300],
                                             name="glove")
                word_emb = tf.concat([word_emb, self._glove], axis=2)

            _, self.dialogue_embedding = rnn.rnn_factory(
                inputs=word_emb,
                seq_length=self._seq_length,
                cell=config['question']["cell"],
                num_hidden=config['question']["rnn_units"],
                bidirectional=config["question"]["bidirectional"],
                max_pool=config["question"]["max_pool"],
                layer_norm=config["question"]["layer_norm"],
                reuse=reuse)

            #####################
            #   IMAGE
            #####################

            self.img_embedding = None
            if config['inputs']['image']:

                self._image = tf.placeholder(tf.float32, [batch_size] +
                                             config['image']["dim"],
                                             name='image')

                # get image
                self.img_embedding = get_image_features(
                    image=self._image,
                    is_training=self._is_training,
                    config=config['image'])

                # pool image feature if needed
                if len(self.img_embedding.get_shape()) > 2:
                    with tf.variable_scope("image_pooling"):
                        self.img_embedding = get_attention(
                            self.img_embedding,
                            self.dialogue_embedding,
                            is_training=self._is_training,
                            config=config["pooling"],
                            dropout_keep=dropout_keep,
                            reuse=reuse)

                # fuse vision/language
                self.visdiag_embedding = get_fusion_mechanism(
                    input1=self.dialogue_embedding,
                    input2=self.img_embedding,
                    config=config.get["fusion"],
                    dropout_keep=dropout_keep)
            else:
                self.visdiag_embedding = self.dialogue_embedding

            visdiag_dim = int(self.visdiag_embedding.get_shape()[-1])

            #####################
            #   OBJECTS
            #####################

            self._num_object = tf.placeholder(tf.int32, [batch_size],
                                              name='obj_seq_length')
            self._obj_cats = tf.placeholder(tf.int32, [batch_size, None],
                                            name='obj_cat')
            self._obj_spats = tf.placeholder(tf.float32, [batch_size, None, 8],
                                             name='obj_spat')

            cats_emb = tfc_layers.embed_sequence(
                ids=self._obj_cats,
                vocab_size=config['category']["n_categories"] +
                1,  # we add the unknown category
                embed_dim=config['category']["embedding_dim"],
                scope="cat_embedding",
                reuse=reuse)
            '''
            spatial_emb = tfc_layers.fully_connected(self._obj_spats,
                                                     num_outputs=config["spatial"]["no_mlp_units"],
                                                     activation_fn=tf.nn.relu,
                                                     reuse=reuse,
                                                     scope="spatial_upsampling")
            '''
            spatial_emb = self._obj_spats

            self.objects_input = tf.concat([cats_emb, spatial_emb], axis=2)
            # self.objects_input = tf.nn.dropout(self.objects_input, dropout_keep)

            with tf.variable_scope('obj_mlp'):
                h1 = tfc_layers.fully_connected(
                    self.objects_input,
                    num_outputs=config["object"]['no_mlp_units'],
                    activation_fn=tf.nn.relu,
                    scope='l1')
                # h1 = tf.nn.dropout(h1, dropout_keep)

                obj_embeddings = tfc_layers.fully_connected(
                    h1,
                    num_outputs=visdiag_dim,
                    activation_fn=tf.nn.relu,
                    scope='l2')

            #####################
            #   SCORES
            #####################

            self.scores = obj_embeddings * tf.expand_dims(
                self.visdiag_embedding, axis=1)
            self.scores = tf.reduce_sum(self.scores, axis=2)

            # remove max for stability (trick)
            self.scores -= tf.reduce_max(self.scores, axis=1, keep_dims=True)

            with tf.variable_scope('object_mask', reuse=reuse):

                object_mask = tf.sequence_mask(self._num_object)
                score_mask_values = float("-inf") * tf.ones_like(self.scores)

                self.score_masked = tf.where(object_mask, self.scores,
                                             score_mask_values)

            #####################
            #   LOSS
            #####################

            # Targets
            self._targets = tf.placeholder(tf.int32, [batch_size],
                                           name="target_index")

            self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels=self._targets, logits=self.score_masked)
            self.loss = tf.reduce_mean(self.loss)

            self.selected_object = tf.argmax(self.score_masked, axis=1)
            self.softmax = tf.nn.softmax(self.score_masked)

            with tf.variable_scope('accuracy'):
                self.accuracy = tf.equal(self.selected_object,
                                         tf.cast(self._targets, tf.int64))
                self.accuracy = tf.reduce_mean(
                    tf.cast(self.accuracy, tf.float32))
Пример #5
0
    def __init__(self, config, num_words, device='', reuse=False):
        AbstractNetwork.__init__(self, "guesser", device=device)

        mini_batch_size = None

        with tf.variable_scope(self.scope_name, reuse=reuse):
            # Misc
            self._is_training = tf.placeholder(tf.bool, name='is_training')
            self._is_dynamic = tf.placeholder(tf.bool, name='is_dynamic')
            batch_size = None

            dropout_keep_scalar = float(config.get("dropout_keep_prob", 1.0))
            dropout_keep = tf.cond(self._is_training,
                                   lambda: tf.constant(dropout_keep_scalar),
                                   lambda: tf.constant(1.0))
            # Objects
            self._num_object = tf.placeholder(tf.int32, [mini_batch_size],
                                              name='obj_seq_length')
            self.obj_mask = tf.sequence_mask(self._num_object,
                                             dtype=tf.float32)
            # self.obj_mask = tf.sequence_mask(self._num_object, maxlen=20, dtype=tf.float32)
            # self.obj_mask = tf.placeholder(tf.float32, [mini_batch_size, None], name='obj_mask')
            self.obj_cats = tf.placeholder(tf.int32, [mini_batch_size, None],
                                           name='obj_cat')
            self.obj_spats = tf.placeholder(
                tf.float32, [mini_batch_size, None, config['spat_dim']],
                name='obj_spat')

            # Targets
            self.targets = tf.placeholder(tf.int32, [mini_batch_size],
                                          name="target_index")

            self.object_cats_emb = utils_v1.get_embedding(
                self.obj_cats,
                config['no_categories'] + 1,
                config['cat_emb_dim'],
                scope='cat_embedding')

            self.objects_input = tf.concat(
                [self.object_cats_emb, self.obj_spats], axis=2)
            self.flat_objects_inp = tf.reshape(
                self.objects_input,
                [-1, config['cat_emb_dim'] + config['spat_dim']])

            with tf.variable_scope('obj_mlp'):
                h1 = utils_v1.fully_connected(self.flat_objects_inp,
                                              n_out=config['obj_mlp_units'],
                                              activation='relu',
                                              scope='l1')
                h2 = utils_v1.fully_connected(h1,
                                              n_out=config['dialog_emb_dim'],
                                              activation='relu',
                                              scope='l2')

            obj_embs = tf.reshape(
                h2,
                [-1, tf.shape(self.obj_cats)[1], config['dialog_emb_dim']])

            #####################
            #   UAQRAH PART
            #####################

            #####################
            #   WORD EMBEDDING
            #####################

            with tf.variable_scope('word_embedding', reuse=reuse):
                self.dialogue_emb_weights = tf.get_variable(
                    "dialogue_embedding_encoder",
                    shape=[
                        num_words, config["dialogue"]["word_embedding_dim"]
                    ],
                    initializer=tf.random_uniform_initializer(-0.08, 0.08))

            #####################
            #   DIALOGUE
            #####################

            self._q_his = tf.placeholder(tf.int32, [batch_size, None, None],
                                         name='q_his')
            # self._q_his_mask = tf.placeholder(tf.float32, [batch_size, None, None], name='q_his_mask')
            self._a_his = tf.placeholder(tf.int32, [batch_size, None, None],
                                         name='a_his')
            self._q_his_lengths = tf.placeholder(tf.int32, [batch_size, None],
                                                 name='q_his_lengths')
            self._q_turn = tf.placeholder(tf.int32, [batch_size],
                                          name='q_turn')
            self._max_turn = tf.placeholder(tf.int32, None, name='max_turn')

            bs = tf.shape(self._q_his)[0]

            self.rnn_cell_word = rnn.create_cell(
                config['dialogue']["rnn_word_units"],
                layer_norm=config["dialogue"]["layer_norm"],
                reuse=tf.AUTO_REUSE,
                cell=config['dialogue']["cell"],
                scope="forward_word")

            self.rnn_cell_context = rnn.create_cell(
                config['dialogue']["rnn_context_units"],
                layer_norm=config["dialogue"]["layer_norm"],
                reuse=tf.AUTO_REUSE,
                cell=config['dialogue']["cell"],
                scope="forward_context")
            self.rnn_cell_pair = rnn.create_cell(
                config['dialogue']["rnn_context_units"],
                layer_norm=config["dialogue"]["layer_norm"],
                reuse=tf.AUTO_REUSE,
                cell=config['dialogue']["cell"],
                scope="forward_pair")

            # ini
            dialogue_last_states_ini = self.rnn_cell_context.zero_state(
                bs, dtype=tf.float32)
            pair_states_ini = self.rnn_cell_pair.zero_state(bs,
                                                            dtype=tf.float32)

            self.max_turn_loop = tf.cond(
                self._is_dynamic, lambda: self._max_turn,
                lambda: tf.constant(5, dtype=tf.int32))
            self.turn_0 = tf.constant(0, dtype=tf.int32)
            self.m_num = tf.Variable(1.)
            self.a_yes_token = tf.constant(0, dtype=tf.int32)
            self.a_no_token = tf.constant(1, dtype=tf.int32)
            self.a_na_token = tf.constant(2, dtype=tf.int32)

            #####################
            #   IMAGES
            #####################

            self._image = tf.placeholder(tf.float32,
                                         [batch_size] + config['image']["dim"],
                                         name='image')  # B, 36, 2048
            if config['image'].get('normalize', False):
                self.img_feature = tf.nn.l2_normalize(self._image,
                                                      dim=-1,
                                                      name="img_normalization")
            else:
                self.img_feature = self._image
            self.vis_dif = OD_compute(self.img_feature)

            self.att_ini = tf.ones([bs, 36])

            def compute_v_dif(att):
                max_obj = tf.argmax(att, axis=1)
                obj_oneh = tf.one_hot(max_obj, depth=36)  # 64,36
                vis_dif_select = tf.boolean_mask(
                    self.vis_dif, obj_oneh)  # 64,36,36*2048 to 64,36*2048
                vis_dif_select = tf.reshape(vis_dif_select, [bs, 36, 2048])
                vis_dif_weighted = tf.reduce_sum(vis_dif_select *
                                                 tf.expand_dims(att, -1),
                                                 axis=1)
                vis_dif_weighted = tf.nn.l2_normalize(
                    vis_dif_weighted, dim=-1, name="v_dif_normalization")
                return vis_dif_weighted

            turn_i = tf.constant(0, dtype=tf.int32)

            def uaqra_att_ini(vis_fea, q_fea, dialogue_state, config,
                              is_training):
                with tf.variable_scope("q_guide_image_pooling"):
                    att_q = compute_current_att(vis_fea,
                                                q_fea,
                                                config,
                                                is_training,
                                                reuse=reuse)
                    att_q = tf.nn.softmax(att_q, axis=-1)

                with tf.variable_scope("h_guide_image_pooling"):
                    att_h = compute_current_att(self.img_feature,
                                                dialogue_state,
                                                config,
                                                is_training,
                                                reuse=reuse)
                    att_h = tf.nn.softmax(att_h, axis=-1)
                att = att_q + att_h
                return att

            def uaqra_att(vis_fea, q_fea, dialogue_state, att_prev, config,
                          answer, m_num, is_training):
                with tf.variable_scope("q_guide_image_pooling"):
                    att_q = compute_current_att(vis_fea,
                                                q_fea,
                                                config,
                                                is_training,
                                                reuse=True)
                f_att_q = cond_gumbel_softmax(is_training, att_q)
                a_list = tf.reshape(answer, shape=[-1, 1])  # is it ok?
                a_list = a_list - 5
                att_na = att_prev
                att_yes = f_att_q * att_prev
                att_no = (1 - f_att_q) * att_prev
                att_select_na = tf.where(a_list == self.a_na_token, att_na,
                                         att_no)
                att_select_yes = tf.where(a_list == self.a_yes_token, att_yes,
                                          att_select_na)
                att_select = att_select_yes
                att_norm = tf.nn.l2_normalize(att_select,
                                              dim=-1,
                                              name="att_normalization")
                att_enlarged = att_norm * m_num
                att_mask = tf.greater(att_enlarged, 0.)
                att_new = maskedSoftmax(att_enlarged, att_mask)
                with tf.variable_scope("h_guide_image_pooling"):
                    att_h = compute_current_att(self.img_feature,
                                                dialogue_state,
                                                config,
                                                is_training,
                                                reuse=True)
                    att_h = tf.nn.softmax(att_h, axis=-1)
                att = att_new + att_h

                return att

            att_list_ini = tf.expand_dims(self.att_ini, 0)
            hpair_list_ini = tf.expand_dims(pair_states_ini, 0)

            def cond_loop(cur_turn, att_prev, dialogue_state, att_list,
                          hpair_list):
                return tf.less(cur_turn, self.max_turn_loop)

            def dialog_flow(cur_turn, att_prev, dialogue_state, att_list,
                            hpair_list):

                #####################
                #   ENCODE CUR_TURN
                #####################

                self._question = self._q_his[:, cur_turn, :]
                self._answer = self._a_his[:, cur_turn]
                self._seq_length_question = self._q_his_lengths[:, cur_turn]

                self.word_emb_question = tf.nn.embedding_lookup(
                    params=self.dialogue_emb_weights, ids=self._question)
                self.word_emb_question = tf.nn.dropout(self.word_emb_question,
                                                       dropout_keep)

                self.word_emb_answer = tf.nn.embedding_lookup(
                    params=self.dialogue_emb_weights, ids=self._answer)
                self.word_emb_answer = tf.nn.dropout(self.word_emb_answer,
                                                     dropout_keep)
                ''' Update the dialog state '''
                seq_mask = tf.cast(tf.greater(self._seq_length_question, 1),
                                   self._seq_length_question.dtype)

                self.outputs_wq, self.h_wq = tf.nn.dynamic_rnn(
                    cell=self.rnn_cell_word,
                    inputs=self.word_emb_question,
                    dtype=tf.float32,
                    sequence_length=self._seq_length_question - 1,
                    scope="forward_word")
                _, self.h_wa = tf.nn.dynamic_rnn(cell=self.rnn_cell_word,
                                                 inputs=self.word_emb_answer,
                                                 dtype=tf.float32,
                                                 sequence_length=seq_mask,
                                                 scope="forward_word")
                _, self.h_c1 = tf.nn.dynamic_rnn(cell=self.rnn_cell_context,
                                                 inputs=tf.expand_dims(
                                                     self.h_wq, 1),
                                                 initial_state=dialogue_state,
                                                 dtype=tf.float32,
                                                 sequence_length=seq_mask,
                                                 scope="forward_context")
                _, dialogue_state = tf.nn.dynamic_rnn(
                    cell=self.rnn_cell_context,
                    inputs=tf.expand_dims(self.h_wa, 1),
                    initial_state=self.h_c1,
                    dtype=tf.float32,
                    sequence_length=seq_mask,
                    scope="forward_context")
                _, self.h_pq = tf.nn.dynamic_rnn(cell=self.rnn_cell_pair,
                                                 inputs=tf.expand_dims(
                                                     self.h_wq, 1),
                                                 dtype=tf.float32,
                                                 scope="forward_pair")
                _, h_pair = tf.nn.dynamic_rnn(cell=self.rnn_cell_pair,
                                              inputs=tf.expand_dims(
                                                  self.h_wa, 1),
                                              initial_state=self.h_pq,
                                              dtype=tf.float32,
                                              scope="forward_pair")

                q_att = compute_q_att(self.outputs_wq, dropout_keep)

                att = tf.cond(
                    tf.equal(cur_turn, self.turn_0), lambda: uaqra_att_ini(
                        self.img_feature, q_att, dialogue_state, config[
                            "pooling"], self._is_training),
                    lambda: uaqra_att(self.img_feature, q_att, dialogue_state,
                                      att_prev, config["pooling"], self.
                                      _answer, self.m_num, self._is_training))

                att_list = tf.cond(
                    tf.equal(cur_turn,
                             self.turn_0), lambda: tf.expand_dims(att, 0),
                    lambda: tf.concat([att_list,
                                       tf.expand_dims(att, 0)], 0))
                hpair_list = tf.cond(
                    tf.equal(cur_turn,
                             self.turn_0), lambda: tf.expand_dims(h_pair, 0),
                    lambda: tf.concat([hpair_list,
                                       tf.expand_dims(h_pair, 0)], 0))

                cur_turn = tf.add(cur_turn, 1)

                return cur_turn, att, dialogue_state, att_list, hpair_list

            _, _, self.dialogue_last_states, self.att_list, self.hpair_list = tf.while_loop(
                cond_loop,
                dialog_flow, [
                    turn_i, self.att_ini, dialogue_last_states_ini,
                    att_list_ini, hpair_list_ini
                ],
                shape_invariants=[
                    turn_i.get_shape(),
                    self.att_ini.get_shape(),
                    dialogue_last_states_ini.get_shape(),
                    tf.TensorShape([None, None, 36]),
                    tf.TensorShape([None, None, 1200])
                ])
            att_list = tf.transpose(self.att_list, perm=[1, 0,
                                                         2])  # 64,max_turn,36
            att_oneh = tf.one_hot(self._q_turn - 1,
                                  depth=self.max_turn_loop)  # 64,max_turn
            self.att = tf.boolean_mask(att_list,
                                       att_oneh)  # 64,max_turn,36 to 64,36
            hpair_list = tf.transpose(self.hpair_list,
                                      [1, 0, 2])  # 64,max_turn,36
            self.h_pair = tf.boolean_mask(hpair_list,
                                          att_oneh)  # 64,max_turn,36 to 64,36
            visual_features = tf.reduce_sum(self.img_feature *
                                            tf.expand_dims(self.att, -1),
                                            axis=1)
            visual_features = tf.nn.l2_normalize(visual_features,
                                                 dim=-1,
                                                 name="v_fea_normalization")
            vis_dif_weighted = compute_v_dif(self.att)

            with tf.variable_scope("compute_beta"):
                self.h_pair = tf.nn.dropout(
                    self.h_pair,
                    dropout_keep)  # considering about the tanh activation
                beta = tfc_layers.fully_connected(self.h_pair,
                                                  num_outputs=2,
                                                  activation_fn=tf.nn.softmax,
                                                  reuse=reuse,
                                                  scope="beta_computation")

            beta_0 = tf.tile(tf.expand_dims(tf.gather(beta, 0, axis=1), 1),
                             [1, 2048])
            beta_1 = tf.tile(tf.expand_dims(tf.gather(beta, 1, axis=1), 1),
                             [1, 2048])
            self.v_final = beta_0 * visual_features + beta_1 * vis_dif_weighted

            with tf.variable_scope("multimodal_fusion"):
                # concat
                self.visdiag_embedding = tfc_layers.fully_connected(
                    tf.concat([self.dialogue_last_states, self.v_final],
                              axis=-1),
                    num_outputs=config['fusion']['projection_size'],
                    activation_fn=tf.nn.tanh,
                    reuse=reuse,
                    scope="visdiag_projection")

            scores = tf.matmul(obj_embs,
                               tf.expand_dims(self.visdiag_embedding, axis=-1))
            scores = tf.reshape(scores, [-1, tf.shape(self.obj_cats)[1]])

            def masked_softmax(scores, mask):
                # subtract max for stability
                scores = scores - tf.tile(
                    tf.reduce_max(scores, axis=(1, ), keep_dims=True),
                    [1, tf.shape(scores)[1]])
                # compute padded softmax
                exp_scores = tf.exp(scores)
                exp_scores *= mask
                exp_sum_scores = tf.reduce_sum(exp_scores,
                                               axis=1,
                                               keep_dims=True)
                return exp_scores / tf.tile(exp_sum_scores,
                                            [1, tf.shape(exp_scores)[1]])

            self.softmax = masked_softmax(scores, self.obj_mask)
            self.selected_object = tf.argmax(self.softmax, axis=1)

            self.loss = tf.reduce_mean(
                utils_v1.cross_entropy(self.softmax, self.targets))
            self.error = tf.reduce_mean(
                utils_v1.error(self.softmax, self.targets))
Пример #6
0
    def __init__(self,
                 config,
                 num_words,
                 rl_module,
                 device='',
                 reuse=tf.AUTO_REUSE):
        AbstractNetwork.__init__(self, "qgen", device=device)

        self.rl_module = rl_module

        # Create the scope for this graph
        with tf.variable_scope(self.scope_name, reuse=reuse):

            # Misc
            self._is_training = tf.placeholder(tf.bool, name='is_training')
            self._is_dynamic = tf.placeholder(tf.bool, name='is_dynamic')
            batch_size = None

            dropout_keep_scalar = float(config.get("dropout_keep_prob", 1.0))
            dropout_keep = tf.cond(self._is_training,
                                   lambda: tf.constant(dropout_keep_scalar),
                                   lambda: tf.constant(1.0))
            # dropout_keep = tf.constant(1.0)

            #####################
            #   WORD EMBEDDING
            #####################

            with tf.variable_scope('word_embedding', reuse=reuse):
                self.dialogue_emb_weights = tf.get_variable(
                    "dialogue_embedding_encoder",
                    shape=[
                        num_words, config["dialogue"]["word_embedding_dim"]
                    ])

            #####################
            #   DIALOGUE
            #####################

            if self.rl_module is not None:
                self._q_flag = tf.placeholder(tf.float32,
                                              shape=[None, batch_size],
                                              name='q_flag')
                self._cum_rewards = tf.placeholder(
                    tf.float32,
                    shape=[batch_size, None, None],
                    name='cum_reward')
                # self._skewness = tf.placeholder(tf.float32, shape=[None, batch_size], name='skewness')
                # self._softmax = tf.placeholder(tf.float32, shape=[None, batch_size, None], name='softmax')
                # self._guess_softmax = tf.transpose(self._softmax, [1, 0, 2])  # batch, turn, o_n
            self._q_his = tf.placeholder(tf.int32, [batch_size, None, None],
                                         name='q_his')
            # self._q_his_mask = tf.placeholder(tf.float32, [batch_size, None, None], name='q_his_mask')
            self._a_his = tf.placeholder(tf.int32, [batch_size, None, None],
                                         name='a_his')
            self._q_his_lengths = tf.placeholder(tf.int32, [batch_size, None],
                                                 name='q_his_lengths')
            self._q_turn = tf.placeholder(tf.int32, None, name='q_turn')

            bs = tf.shape(self._q_his)[0]

            self.rnn_cell_word = rnn.create_cell(
                config['dialogue']["rnn_word_units"],
                layer_norm=config["dialogue"]["layer_norm"],
                reuse=tf.AUTO_REUSE,
                cell=config['dialogue']["cell"],
                scope="forward_word")
            # self.word_init_states = rnn_cell_word.zero_state(batch_size, dtype=tf.float32)

            self.rnn_cell_context = rnn.create_cell(
                config['dialogue']["rnn_context_units"],
                layer_norm=config["dialogue"]["layer_norm"],
                reuse=tf.AUTO_REUSE,
                cell=config['dialogue']["cell"],
                scope="forward_context")
            self.rnn_cell_pair = rnn.create_cell(
                config['dialogue']["rnn_context_units"],
                layer_norm=config["dialogue"]["layer_norm"],
                reuse=tf.AUTO_REUSE,
                cell=config['dialogue']["cell"],
                scope="forward_pair")

            # ini
            dialogue_last_states_ini = self.rnn_cell_context.zero_state(
                bs, dtype=tf.float32)
            pair_states_ini = self.rnn_cell_pair.zero_state(bs,
                                                            dtype=tf.float32)
            q_ini = tf.nn.embedding_lookup(params=self.dialogue_emb_weights,
                                           ids=tf.fill([bs], num_words))
            outputs_w_ini, _ = tf.nn.dynamic_rnn(cell=self.rnn_cell_word,
                                                 inputs=tf.expand_dims(q_ini,
                                                                       axis=1),
                                                 dtype=tf.float32,
                                                 scope="forward_word")
            qatt_ini = compute_q_att(outputs_w_ini, dropout_keep)
            answer_ini = self._a_his[:, 0]

            assert config['decoder'][
                "cell"] != "lstm", "LSTM are not yet supported for the decoder"
            self.decoder_cell = rnn.create_cell(
                cell=config['decoder']["cell"],
                num_units=config['fusion']['projection_size'],
                layer_norm=config["decoder"]["layer_norm"],
                reuse=reuse)

            self.decoder_projection_layer = tf.layers.Dense(num_words - 1)

            loss_ini = 0.

            self.q_turn_loop = tf.cond(self._is_dynamic, lambda: self._q_turn,
                                       lambda: tf.constant(5, dtype=tf.int32))
            self.turn_0 = tf.constant(0, dtype=tf.int32)
            self.m_num = tf.Variable(1.)
            self.a_yes_token = tf.constant(0, dtype=tf.int32)
            self.a_no_token = tf.constant(1, dtype=tf.int32)
            self.a_na_token = tf.constant(2, dtype=tf.int32)

            #####################
            #   IMAGES
            #####################

            self._image = tf.placeholder(tf.float32,
                                         [batch_size] + config['image']["dim"],
                                         name='image')  # B, 36, 2048
            if config['image'].get('normalize', False):
                self.img_feature = tf.nn.l2_normalize(self._image,
                                                      dim=-1,
                                                      name="fc_normalization")
            self.vis_dif = OD_compute(self._image)

            # Pool Image Features
            with tf.variable_scope("q_guide_image_pooling"):
                att_ini_q = compute_current_att(self.img_feature,
                                                qatt_ini,
                                                config["pooling"],
                                                self._is_training,
                                                reuse=reuse)
                att_ini_q = tf.nn.softmax(att_ini_q, axis=-1)

            with tf.variable_scope("h_guide_image_pooling"):
                att_ini_h = compute_current_att(self.img_feature,
                                                dialogue_last_states_ini,
                                                config["pooling"],
                                                self._is_training,
                                                reuse=reuse)
                att_ini_h = tf.nn.softmax(att_ini_h, axis=-1)

            self.att_ini = att_ini_h + att_ini_q
            visual_features_ini = tf.reduce_sum(
                self.img_feature * tf.expand_dims(self.att_ini, -1), axis=1)
            visual_features_ini = tf.nn.l2_normalize(
                visual_features_ini, dim=-1, name="v_fea_normalization")

            def compute_v_dif(att):
                max_obj = tf.argmax(att, axis=1)
                obj_oneh = tf.one_hot(max_obj, depth=36)  # 64,36
                vis_dif_select = tf.boolean_mask(
                    self.vis_dif, obj_oneh)  # 64,36,36*2048 to 64,36*2048
                vis_dif_select = tf.reshape(vis_dif_select, [bs, 36, 2048])
                vis_dif_weighted = tf.reduce_sum(vis_dif_select *
                                                 tf.expand_dims(att, -1),
                                                 axis=1)
                vis_dif_weighted = tf.nn.l2_normalize(
                    vis_dif_weighted, dim=-1, name="v_dif_normalization")
                return vis_dif_weighted

            vis_dif_weighted_ini = compute_v_dif(self.att_ini)

            with tf.variable_scope("compute_beta"):
                pair_states_ini = tf.nn.dropout(pair_states_ini, dropout_keep)
                beta_ini = tfc_layers.fully_connected(
                    pair_states_ini,
                    num_outputs=2,
                    activation_fn=tf.nn.softmax,
                    reuse=reuse,
                    scope="beta_computation")
            beta_0_ini = tf.tile(
                tf.expand_dims(tf.gather(beta_ini, 0, axis=1), 1), [1, 2048])
            beta_1_ini = tf.tile(
                tf.expand_dims(tf.gather(beta_ini, 1, axis=1), 1), [1, 2048])

            v_final_ini = beta_0_ini * visual_features_ini + beta_1_ini * vis_dif_weighted_ini

            turn_i = 0
            totals_ini = 0.

            with tf.variable_scope("multimodal_fusion"):

                visdiag_embedding_ini = tfc_layers.fully_connected(
                    tf.concat([dialogue_last_states_ini, v_final_ini],
                              axis=-1),
                    num_outputs=config['fusion']['projection_size'],
                    activation_fn=tf.nn.tanh,
                    reuse=reuse,
                    scope="visdiag_projection")

            def uaqra_att(vis_fea, q_fea, dialogue_state, h_pair, att_prev,
                          config, answer, m_num, is_training):
                with tf.variable_scope("q_guide_image_pooling"):
                    att_q = compute_current_att(vis_fea,
                                                q_fea,
                                                config,
                                                is_training,
                                                reuse=True)
                f_att_q = cond_gumbel_softmax(is_training, att_q)
                a_list = tf.reshape(answer, shape=[-1, 1])  # is it ok?
                a_list = a_list - 5
                att_na = att_prev
                att_yes = f_att_q * att_prev
                att_no = (1 - f_att_q) * att_prev
                att_select_na = tf.where(a_list == self.a_na_token, att_na,
                                         att_no)
                att_select_yes = tf.where(a_list == self.a_yes_token, att_yes,
                                          att_select_na)
                att_select = att_select_yes
                att_norm = tf.nn.l2_normalize(att_select,
                                              dim=-1,
                                              name="att_normalization")
                att_enlarged = att_norm * m_num
                att_mask = tf.greater(att_enlarged, 0.)
                att_new = maskedSoftmax(att_enlarged, att_mask)
                with tf.variable_scope("h_guide_image_pooling"):
                    att_h = compute_current_att(self.img_feature,
                                                dialogue_state,
                                                config,
                                                is_training,
                                                reuse=True)
                    att_h = tf.nn.softmax(att_h, axis=-1)
                att = att_new + att_h

                visual_features = tf.reduce_sum(self.img_feature *
                                                tf.expand_dims(att, -1),
                                                axis=1)
                visual_features = tf.nn.l2_normalize(
                    visual_features, dim=-1, name="v_fea_normalization")
                vis_dif_weighted = compute_v_dif(att)

                with tf.variable_scope("compute_beta"):
                    h_pair = tf.nn.dropout(
                        h_pair,
                        dropout_keep)  # considering about the tanh activation
                    beta = tfc_layers.fully_connected(
                        h_pair,
                        num_outputs=2,
                        activation_fn=tf.nn.softmax,
                        reuse=True,
                        scope="beta_computation")

                beta_0 = tf.tile(tf.expand_dims(tf.gather(beta, 0, axis=1), 1),
                                 [1, 2048])
                beta_1 = tf.tile(tf.expand_dims(tf.gather(beta, 1, axis=1), 1),
                                 [1, 2048])
                v_final = beta_0 * visual_features + beta_1 * vis_dif_weighted

                return att, v_final, beta

            def cond_loop(cur_turn, loss, totals, q_att, h_pair, answer,
                          att_prev, dialogue_state, visual_features,
                          visdiag_embedding, beta):
                # print_info = tf.Print(cur_turn, [cur_turn], "x:")
                # cur_turn = cur_turn + print_info
                return tf.less(cur_turn, self.q_turn_loop)

            def dialog_flow(cur_turn, loss, totals, q_att, h_pair, answer,
                            att_prev, dialogue_state, visual_features,
                            visdiag_embedding, beta):

                att, v_final, beta = tf.cond(
                    tf.equal(cur_turn, self.turn_0), lambda:
                    (self.att_ini, visual_features, beta_ini),
                    lambda: uaqra_att(self.img_feature, q_att, dialogue_state,
                                      h_pair, att_prev, config["pooling"],
                                      answer, self.m_num, self._is_training))
                repeat_v_final = tf.tile(tf.expand_dims(v_final, axis=1),
                                         [1, 12, 1])

                with tf.variable_scope("multimodal_fusion"):
                    # concat
                    visdiag_embedding = tfc_layers.fully_connected(
                        tf.concat([dialogue_state, v_final], axis=-1),
                        num_outputs=config['fusion']['projection_size'],
                        activation_fn=tf.nn.tanh,
                        reuse=True,
                        scope="visdiag_projection")

                #####################
                #   TARGET QUESTION
                #####################

                self._question = self._q_his[:, cur_turn, :]
                self._answer = self._a_his[:, cur_turn]
                self._seq_length_question = self._q_his_lengths[:, cur_turn]
                self._mask = tf.sequence_mask(
                    lengths=self._seq_length_question - 1,
                    maxlen=12,
                    dtype=tf.float32)

                self.word_emb_question = tf.nn.embedding_lookup(
                    params=self.dialogue_emb_weights, ids=self._question)
                self.word_emb_question = tf.nn.dropout(self.word_emb_question,
                                                       dropout_keep)

                self.word_emb_question_input = self.word_emb_question[:, :
                                                                      -1, :]
                self.word_emb_question_encode = self.word_emb_question[:,
                                                                       1:, :]

                self.word_emb_answer = tf.nn.embedding_lookup(
                    params=self.dialogue_emb_weights, ids=self._answer)
                self.word_emb_answer = tf.nn.dropout(self.word_emb_answer,
                                                     dropout_keep)

                #####################
                #   DECODER
                #####################

                self.decoder_states, _ = tf.nn.dynamic_rnn(
                    cell=self.decoder_cell,
                    inputs=tf.concat(
                        [self.word_emb_question_input, repeat_v_final],
                        axis=-1),
                    dtype=tf.float32,
                    initial_state=visdiag_embedding,
                    sequence_length=self._seq_length_question - 1,
                    scope="decoder")

                self.decoder_outputs = self.decoder_projection_layer(
                    self.decoder_states)

                #####################
                #   LOSS
                #####################

                # compute the softmax for evaluation
                ''' Compute policy gradient '''
                if self.rl_module is not None:

                    # Step 1: compute the state-value function
                    self._cum_rewards_current = self._cum_rewards[:, cur_turn,
                                                                  1:]
                    self._cum_rewards_current *= self._mask

                    # q cost
                    # self._q_flag_cur = self._q_flag[cur_turn, :]

                    # guess softmax
                    # self._guess_cur = self._guess_softmax[:, cur_turn+1, :]
                    # self._guess_pre = self._guess_softmax[:, cur_turn, :]
                    # skewness
                    # self._skewness_cur = self._skewness[cur_turn, :]
                    # Step 2: compute the state-value function
                    value_state = self.decoder_states
                    if self.rl_module.stop_gradient:
                        value_state = tf.stop_gradient(self.decoder_states)
                    # value_state = tf.nn.dropout(value_state, dropout_keep)
                    v_num_hidden_units = int(
                        int(value_state.get_shape()[-1]) / 4)

                    with tf.variable_scope('value_function'):
                        self.value_function = tf.keras.models.Sequential()
                        # self.value_function.add(tf.layers.Dropout(rate=dropout_keep))
                        self.value_function.add(
                            tf.layers.Dense(
                                units=v_num_hidden_units,
                                activation=tf.nn.relu,
                                input_shape=(int(
                                    value_state.get_shape()[-1]), ),
                                name="value_function_hidden"))
                        # self.value_function.add(tf.layers.Dropout(rate=dropout_keep))
                        self.value_function.add(
                            tf.layers.Dense(units=1,
                                            activation=None,
                                            name="value_function"))
                        self.value_function.add(tf.keras.layers.Reshape(
                            (-1, )))

                    # Step 3: compute the RL loss (reinforce, A3C, PPO etc.)
                    loss_i = rl_module(
                        cum_rewards=self._cum_rewards_current,
                        value_function=self.value_function(value_state),
                        policy_state=self.decoder_outputs,
                        actions=self._question[:, 1:],
                        action_mask=self._mask)

                    # q_flag=self._q_flag_cur,
                    # pre_g=self._guess_pre,
                    # cur_g=self._guess_cur,
                    # skew=self._skewness_cur)

                    loss += loss_i
                    totals = 1.0

                else:
                    '''supervised loss'''
                    with tf.variable_scope('ml_loss'):
                        cr_loss = tfc_seq.sequence_loss(
                            logits=self.decoder_outputs,
                            targets=self._question[:, 1:],
                            weights=self._mask,
                            average_across_timesteps=False,
                            average_across_batch=False)

                        # cr_loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.decoder_outputs,
                        #                                                          labels=self._question[:, 1:])
                        ml_loss = tf.identity(cr_loss)
                        ml_loss *= self._mask
                        ml_loss = tf.reduce_sum(
                            ml_loss, axis=1)  # reduce over question dimension
                        ml_loss = tf.reduce_sum(
                            ml_loss, axis=0)  # reduce over minibatch dimension
                        count = tf.reduce_sum(self._mask)
                        self.softmax_output = tf.nn.softmax(
                            self.decoder_outputs, name="softmax")
                        self.argmax_output = tf.argmax(self.decoder_outputs,
                                                       axis=2)
                        loss += ml_loss
                        totals += count
                ''' Update the dialog state '''
                self.outputs_wq, self.h_wq = tf.nn.dynamic_rnn(
                    cell=self.rnn_cell_word,
                    inputs=self.word_emb_question_encode,
                    dtype=tf.float32,
                    sequence_length=self._seq_length_question - 1,
                    scope="forward_word")
                _, self.h_wa = tf.nn.dynamic_rnn(cell=self.rnn_cell_word,
                                                 inputs=self.word_emb_answer,
                                                 dtype=tf.float32,
                                                 scope="forward_word")
                _, self.h_c1 = tf.nn.dynamic_rnn(cell=self.rnn_cell_context,
                                                 inputs=tf.expand_dims(
                                                     self.h_wq, 1),
                                                 initial_state=dialogue_state,
                                                 dtype=tf.float32,
                                                 scope="forward_context")
                _, dialogue_state = tf.nn.dynamic_rnn(
                    cell=self.rnn_cell_context,
                    inputs=tf.expand_dims(self.h_wa, 1),
                    initial_state=self.h_c1,
                    dtype=tf.float32,
                    scope="forward_context")
                _, self.h_pq = tf.nn.dynamic_rnn(cell=self.rnn_cell_pair,
                                                 inputs=tf.expand_dims(
                                                     self.h_wq, 1),
                                                 dtype=tf.float32,
                                                 scope="forward_pair")
                _, h_pair = tf.nn.dynamic_rnn(cell=self.rnn_cell_pair,
                                              inputs=tf.expand_dims(
                                                  self.h_wa, 1),
                                              initial_state=self.h_pq,
                                              dtype=tf.float32,
                                              scope="forward_pair")

                q_att = compute_q_att(self.outputs_wq, dropout_keep)

                cur_turn = tf.add(cur_turn, 1)

                return cur_turn, loss, totals, q_att, h_pair, self._answer, att, dialogue_state, v_final, visdiag_embedding, beta

            _, loss_f, totals_f, _, _, _, self.att, self.dialogue_last_states, self.visual_features, self.visdiag_embedding, self.beta = tf.while_loop(
                cond_loop, dialog_flow, [
                    turn_i, loss_ini, totals_ini, qatt_ini, pair_states_ini,
                    answer_ini, self.att_ini, dialogue_last_states_ini,
                    visual_features_ini, visdiag_embedding_ini, beta_ini
                ])
            self.loss = loss_f / totals_f