示例#1
0
    def __init__(self,
                 is_training,
                 learning_rate=1.0,
                 optimizer="sgd",
                 max_grad_norm=5,
                 num_layers=2,
                 use_lstm=True,
                 num_steps=35,
                 num_steps_valid=120,
                 proj_size=650,
                 hidden_size=650,
                 hidden_proj=650,
                 num_samples=512,
                 init_scale=0.1,
                 dropout_rate=0.0,
                 lr_decay=0.8,
                 batch_size=20,
                 attentive=False,
                 projection_attention_f=None,
                 output_form=lm_ops.OUTPUT_CONCAT,
                 vocab_size=10000):

        with tf.device("/gpu:0"):

            if attentive:
                assert projection_attention_f is not None

            self.batch_size = batch_size = batch_size
            self.num_steps = num_steps
            self.num_steps_valid = num_steps_valid
            vocab_size = vocab_size

            self._input_data_train = []
            self._targets_train = []
            self.mask_train = []

            for i in xrange(num_steps):  # Last bucket is the biggest one.
                self.input_data_train.append(
                    tf.placeholder(tf.int32,
                                   shape=[None],
                                   name="input_train{0}".format(i)))
                self.targets_train.append(
                    tf.placeholder(tf.int32,
                                   shape=[None],
                                   name="target_train{0}".format(i)))
                self.mask_train.append(
                    tf.placeholder(tf.float32,
                                   shape=[None],
                                   name="mask_train{0}".format(i)))

            self._input_data_valid = []
            self._targets_valid = []
            self.mask_valid = []

            for i in xrange(
                    num_steps_valid):  # Last bucket is the biggest one.
                self.input_data_valid.append(
                    tf.placeholder(tf.int32,
                                   shape=[None],
                                   name="input_valid{0}".format(i)))
                self.targets_valid.append(
                    tf.placeholder(tf.int32,
                                   shape=[None],
                                   name="target_valid{0}".format(i)))
                self.mask_valid.append(
                    tf.placeholder(tf.float32,
                                   shape=[None],
                                   name="mask_valid{0}".format(i)))

            hidden_projection = None
            if hidden_proj > 0:
                hidden_projection = hidden_proj

            self.cell = cells.build_lm_multicell_rnn(
                num_layers,
                hidden_size,
                proj_size,
                use_lstm=use_lstm,
                hidden_projection=hidden_projection,
                dropout=dropout_rate)

            self.dropout_feed = tf.placeholder(tf.float32, name="dropout_rate")

            self._initial_state_train = self.cell.zero_state(
                batch_size, tf.float32)
            self._initial_state_valid = self.cell.zero_state(1, tf.float32)

            # learning rate ops
            self.learning_rate = tf.Variable(float(learning_rate),
                                             trainable=False)
            self.learning_rate_decay_op = self.learning_rate.assign(
                self.learning_rate * lr_decay)

            # epoch ops
            self.epoch = tf.Variable(0, trainable=False)
            self.epoch_update_op = self.epoch.assign(self.epoch + 1)

            # samples seen ops
            self.samples_seen = tf.Variable(0, trainable=False)
            self.samples_seen_update_op = self.samples_seen.assign(
                self.samples_seen + batch_size)
            self.samples_seen_reset_op = self.samples_seen.assign(0)

            # global step variable - controled by the model
            self.global_step = tf.Variable(0.0, trainable=False)

            # average loss ops
            self.current_ppx = tf.Variable(1.0, trainable=False)
            self.current_loss = tf.Variable(0.0, trainable=False)
            # self.current_loss_update_op = None

            self.best_eval_ppx = tf.Variable(numpy.inf, trainable=False)
            self.estop_counter = tf.Variable(0, trainable=False)
            self.estop_counter_update_op = self.estop_counter.assign(
                self.estop_counter + 1)
            self.estop_counter_reset_op = self.estop_counter.assign(0)

            initializer = tf.random_uniform_initializer(minval=init_scale,
                                                        maxval=init_scale,
                                                        seed=_SEED)

            out_proj = hidden_size
            if hidden_proj > 0:
                out_proj = hidden_proj

            with tf.device("/cpu:0"):
                w = tf.get_variable("proj_w", [out_proj, vocab_size])
                w_t = tf.transpose(w)
                b = tf.get_variable("proj_b", [vocab_size])
            self.output_projection = (w, b)

            sampled_softmax = False

            # Sampled softmax only makes sense if we sample less than vocabulary size.
            if 0 < num_samples < vocab_size:
                sampled_softmax = True

                def sampled_loss(logits, labels):
                    with tf.device("/cpu:0"):
                        labels = tf.reshape(labels, [-1, 1])
                        losses = tf.nn.sampled_softmax_loss(
                            w_t, b, logits, labels, num_samples, vocab_size)
                        return losses

                loss_function = sampled_loss

            with tf.device("/cpu:0"):
                # input come as one big tensor so we have to split it into a list of tensors to run the rnn cell
                embedding = tf.Variable(tf.random_uniform(
                    [vocab_size, proj_size],
                    minval=-init_scale,
                    maxval=init_scale),
                                        name="embedding")
                # embedding = tf.get_variable("embedding", [vocab_size, proj_size])

                inputs_train = [
                    tf.nn.embedding_lookup(embedding, i)
                    for i in self.input_data_train
                ]
                inputs_valid = [
                    tf.nn.embedding_lookup(embedding, i)
                    for i in self.input_data_valid
                ]

            with tf.variable_scope("RNN", initializer=initializer):

                if attentive:
                    outputs_train, state_train, _ = lm_ops.apply_attentive_lm(
                        self.cell,
                        inputs_train,
                        sequence_length=array_ops.squeeze(
                            math_ops.add_n(self.mask_train)),
                        projection_attention_f=projection_attention_f,
                        output_form=output_form,
                        dropout=self.dropout_feed,
                        initializer=initializer,
                        dtype=tf.float32)

                    outputs_valid, state_valid, _ = lm_ops.apply_attentive_lm(
                        self.cell,
                        inputs_valid,
                        sequence_length=array_ops.squeeze(
                            math_ops.add_n(self.mask_valid)),
                        projection_attention_f=projection_attention_f,
                        output_form=output_form,
                        dropout=self.dropout_feed,
                        initializer=initializer,
                        dtype=tf.float32)

                else:
                    outputs_train, state_train = lm_ops.apply_lm(
                        self.cell,
                        inputs_train,
                        sequence_length=math_ops.add_n(self.mask_train),
                        dropout=self.dropout_feed,
                        dtype=tf.float32)

                    outputs_valid, state_valid = lm_ops.apply_lm(
                        self.cell,
                        inputs_valid,
                        sequence_length=math_ops.add_n(self.mask_valid),
                        dropout=self.dropout_feed,
                        dtype=tf.float32)

                if sampled_softmax is False:
                    logits_train = [
                        tf.nn.xw_plus_b(o, self.output_projection[0],
                                        self.output_projection[1])
                        for o in outputs_train
                    ]
                    logits_valid = [
                        tf.nn.xw_plus_b(o, self.output_projection[0],
                                        self.output_projection[1])
                        for o in outputs_valid
                    ]
                else:
                    logits_train = outputs_train
                    logits_valid = outputs_valid

            loss_train = seq2seq.sequence_loss_by_example(
                logits_train,
                self.targets_train,
                self.mask_train,
                average_across_timesteps=True)

            loss_valid = seq2seq.sequence_loss_by_example(
                logits_valid,
                self.targets_valid,
                self.mask_valid,
                average_across_timesteps=True)

            self._cost_train = cost = tf.reduce_sum(loss_train) / float(
                batch_size)
            self._final_state_train = state_train

            self._cost_valid = tf.reduce_sum(loss_valid) / float(batch_size)
            self._final_state_valid = state_valid

            if not is_training:
                return

            tvars = tf.trainable_variables()
            grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
                                              max_grad_norm)

            opt = optimization_ops.get_optimizer(optimizer, learning_rate)
            self._train_op = opt.apply_gradients(zip(grads, tvars),
                                                 global_step=self.global_step)
            self._valid_op = tf.no_op()

            self.saver = tf.train.Saver(tf.all_variables())
            self.saver_best = tf.train.Saver(tf.all_variables())
示例#2
0
    def __init__(self,
                 source_vocab_size,
                 target_vocab_size,
                 buckets,
                 source_proj_size,
                 target_proj_size,
                 encoder_size,
                 decoder_size,
                 max_gradient_norm,
                 batch_size,
                 learning_rate,
                 learning_rate_decay_factor,
                 optimizer='sgd',
                 input_feeding=False,
                 combine_inp_attn=False,
                 dropout=0.0,
                 attention_f=None,
                 window_size=10,
                 content_function=None,
                 decoder_attention_f="None",
                 num_samples=512,
                 forward_only=False,
                 max_len=100,
                 cpu_only=False,
                 early_stop_patience=0,
                 save_best_model=True,
                 dtype=tf.float32):
        super(NMTModel, self).__init__()

        if cpu_only:
            device = "/cpu:0"
        else:
            device = "/gpu:0"

        with tf.device(device):

            self.source_vocab_size = source_vocab_size
            self.target_vocab_size = target_vocab_size
            self.buckets = buckets
            self.batch_size = batch_size
            self.attention_f = attention_f
            self.content_function = content_function
            self.window_size = window_size

            self.combine_inp_attn = combine_inp_attn

            if decoder_attention_f == "None":
                self.decoder_attention_f = None
            else:
                self.decoder_attention_f = decoder_attention_f

            # learning rate ops
            self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
            self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * learning_rate_decay_factor)

            # epoch ops
            self.epoch = tf.Variable(0, trainable=False)
            self.epoch_update_op = self.epoch.assign(self.epoch + 1)

            # samples seen ops
            self.samples_seen = tf.Variable(0, trainable=False)
            self.samples_seen_update_op = self.samples_seen.assign(self.samples_seen + batch_size)
            self.samples_seen_reset_op = self.samples_seen.assign(0)

            # global step variable - controled by the model
            self.global_step = tf.Variable(0.0, trainable=False)

            # average loss ops
            self.current_loss = tf.Variable(0.0, trainable=False)
            self.current_loss_update_op = None
            self.avg_loss = tf.Variable(0.0, trainable=False)
            self.avg_loss_update_op = self.avg_loss.assign(tf.div(self.current_loss, self.global_step))

            if early_stop_patience > 0 or save_best_model:
                self.best_eval_loss = tf.Variable(numpy.inf, trainable=False)
                self.estop_counter = tf.Variable(0, trainable=False)
                self.estop_counter_update_op = self.estop_counter.assign(self.estop_counter + 1)
                self.estop_counter_reset_op = self.estop_counter.assign(0)
            else:
                self.best_eval_loss = None
                self.estop_counter = None
                self.estop_counter_update_op = None
                self.estop_counter_reset_op = None

            self.source_proj_size = source_proj_size
            self.target_proj_size = target_proj_size
            self.encoder_size = encoder_size
            self.decoder_size = decoder_size

            self.input_feeding = input_feeding

            self.max_len = max_len
            self.dropout = dropout
            self.dropout_feed = tf.placeholder(tf.float32, name="dropout_rate")
            self.step_num = tf.Variable(0, trainable=False)

            self.dtype = dtype

            # If we use sampled softmax, we need an output projection.
            loss_function = None

            with tf.device("/cpu:0"):
                w = tf.get_variable("proj_w", [decoder_size, self.target_vocab_size])
                w_t = tf.transpose(w)
                b = tf.get_variable("proj_b", [self.target_vocab_size])
            self.output_projection = (w, b)

            self.sampled_softmax = False

            # Sampled softmax only makes sense if we sample less than vocabulary size.
            if 0 < num_samples < self.target_vocab_size:
                self.sampled_softmax = True
                def sampled_loss(inputs, labels):
                    with tf.device("/cpu:0"):
                        labels = tf.reshape(labels, [-1, 1])
                        return tf.nn.sampled_softmax_loss(w_t, b, inputs, labels, num_samples,
                                                          self.target_vocab_size)

                loss_function = sampled_loss

            # create the embedding matrix - this must be done in the CPU for now
            with tf.device("/cpu:0"):
                self.src_embedding = tf.Variable(
                    tf.truncated_normal(
                        [source_vocab_size, source_proj_size], stddev=0.01
                    ),
                    name='embedding_src'
                )

                # decoder with attention
                with tf.name_scope('decoder_with_attention') as scope:
                    # create this variable to be used inside the embedding_attention_decoder
                    self.tgt_embedding = tf.Variable(
                        tf.truncated_normal(
                            [target_vocab_size, target_proj_size], stddev=0.01
                        ),
                        name='embedding'
                    )

            # Create the internal multi-layer cell for our RNN.
            self.encoder_cell_fw, self.encoder_cell_bw, self.decoder_cell = cells.build_nmt_bidirectional_cell(
                encoder_size, decoder_size, source_proj_size, target_proj_size, dropout=dropout)

            # The seq2seq function: we use embedding for the input and attention.
            def seq2seq_f(encoder_inputs, decoder_inputs):
                return self.inference(encoder_inputs, decoder_inputs)

            # Feeds for inputs.
            self.encoder_inputs = []
            self.decoder_inputs = []
            self.target_weights = []

            for i in xrange(buckets[-1][0]):  # Last bucket is the biggest one.
                self.encoder_inputs.append(tf.placeholder(tf.int32, shape=[None], name="encoder{0}".format(i)))

            for i in xrange(buckets[-1][1] + 1):
                self.decoder_inputs.append(tf.placeholder(tf.int32, shape=[None, ], name="decoder{0}".format(i)))
                self.target_weights.append(tf.placeholder(tf.float32, shape=[None], name="weight{0}".format(i)))

            # Our targets are decoder inputs shifted by one.
            targets = [self.decoder_inputs[i + 1]
                       for i in xrange(len(self.decoder_inputs) - 1)]

            self.decoder_states_holders = None

            # Training outputs and losses.
            if forward_only:

                # self.batch_size = beam_size

                for i in xrange(len(self.encoder_inputs), self.max_len):
                    self.encoder_inputs.append(tf.placeholder(tf.int32, shape=[None], name="encoder{0}".format(i)))

                b_size = array_ops.shape(self.encoder_inputs[0])[0]

                # context, decoder_initial_state, attention_states, input_length
                self.ret0, self.ret1, self.ret2 = self.encode(self.encoder_inputs, b_size)

                self.decoder_init_plcholder = tf.placeholder(tf.float32,
                                                             shape=[None, (target_proj_size) * 2],
                                                             name="decoder_init")

                # shape of this placeholder: the first None indicate the batch size and the second the input length
                self.attn_plcholder = tf.placeholder(tf.float32,
                                                     shape=[None, self.ret2.get_shape()[1], target_proj_size],
                                                     name="attention_states")

                # decoder_states = None
                if self.decoder_attention_f is not None:
                    self.decoder_states_holders = tf.placeholder(tf.float32, shape=[None, None, 1, decoder_size],
                                                                 name="decoder_state")
                decoder_states = self.decoder_states_holders

                self.logits, self.states = attention_decoder_nmt(
                    decoder_inputs=[self.decoder_inputs[0]], initial_state=self.decoder_init_plcholder,
                    attention_states=self.attn_plcholder, cell=self.decoder_cell,
                    num_symbols=target_vocab_size, attention_f=attention_f,
                    window_size=window_size, content_function=content_function,
                    decoder_attention_f=decoder_attention_f, combine_inp_attn=combine_inp_attn,
                    input_feeding=input_feeding, dropout=self.dropout_feed, initializer=None,
                    dtype=dtype
                )

                # If we use output projection, we need to project outputs for decoding.
                self.logits = tf.nn.xw_plus_b(self.logits[-1], self.output_projection[0], self.output_projection[1])
                self.logits = nn_ops.softmax(self.logits)

            else:

                tf_version = pkg_resources.get_distribution("tensorflow").version

                if tf_version == "0.6.0" or tf_version == "0.5.0":

                    self.outputs, self.losses = seq2seq.model_with_buckets(
                        encoder_inputs=self.encoder_inputs, decoder_inputs=self.decoder_inputs,
                        targets=targets, weights=self.target_weights, num_decoder_symbols=self.target_vocab_size,
                        buckets=buckets, seq2seq=lambda x, y: seq2seq_f(x, y), softmax_loss_function=loss_function)

                else:

                    self.outputs, self.losses = model_with_buckets(
                        encoder_inputs=self.encoder_inputs, decoder_inputs=self.decoder_inputs,
                        targets=targets, weights=self.target_weights, buckets=buckets,
                        seq2seq_f=lambda x, y: seq2seq_f(x, y), softmax_loss_function=loss_function)

            # Gradients and SGD update operation for training the model.
            params = tf.trainable_variables()
            if not forward_only:
                self.gradient_norms = []
                self.updates = []
                # opt = tf.train.GradientDescentOptimizer(self.learning_rate)
                opt = optimization_ops.get_optimizer(optimizer, learning_rate)
                for b in xrange(len(buckets)):
                    gradients = tf.gradients(self.losses[b], params)
                    clipped_gradients, norm = tf.clip_by_global_norm(gradients,
                                                                     max_gradient_norm)
                    self.gradient_norms.append(norm)
                    self.updates.append(opt.apply_gradients(
                        zip(clipped_gradients, params), global_step=self.global_step))

            self.saver = tf.train.Saver(tf.all_variables())
            self.saver_best = tf.train.Saver(tf.all_variables())
示例#3
0
    def __init__(self,
                 is_training,
                 learning_rate=1.0,
                 optimizer="sgd",
                 max_grad_norm=5,
                 num_layers=2,
                 use_lstm=True,
                 num_steps=35,
                 num_steps_valid=120,
                 proj_size=650,
                 hidden_size=650,
                 hidden_proj=650,
                 num_samples=512,
                 init_scale=0.1,
                 dropout_rate=0.0,
                 lr_decay=0.8,
                 batch_size=20,
                 attentive=False,
                 projection_attention_f=None,
                 output_form=lm_ops.OUTPUT_CONCAT,
                 vocab_size=10000):

        with tf.device("/gpu:0"):

            if attentive:
                assert projection_attention_f is not None

            self.batch_size = batch_size = batch_size
            self.num_steps = num_steps
            self.num_steps_valid = num_steps_valid
            vocab_size = vocab_size

            self._input_data_train = []
            self._targets_train = []
            self.mask_train = []

            for i in xrange(num_steps):  # Last bucket is the biggest one.
                self.input_data_train.append(tf.placeholder(tf.int32, shape=[None], name="input_train{0}".format(i)))
                self.targets_train.append(tf.placeholder(tf.int32, shape=[None], name="target_train{0}".format(i)))
                self.mask_train.append(tf.placeholder(tf.float32, shape=[None], name="mask_train{0}".format(i)))

            self._input_data_valid = []
            self._targets_valid = []
            self.mask_valid = []

            for i in xrange(num_steps_valid):  # Last bucket is the biggest one.
                self.input_data_valid.append(tf.placeholder(tf.int32, shape=[None], name="input_valid{0}".format(i)))
                self.targets_valid.append(tf.placeholder(tf.int32, shape=[None], name="target_valid{0}".format(i)))
                self.mask_valid.append(tf.placeholder(tf.float32, shape=[None], name="mask_valid{0}".format(i)))

            hidden_projection = None
            if hidden_proj > 0:
                hidden_projection = hidden_proj

            self.cell = cells.build_lm_multicell_rnn(num_layers, hidden_size, proj_size, use_lstm=use_lstm,
                                                     hidden_projection=hidden_projection, dropout=dropout_rate)

            self.dropout_feed = tf.placeholder(tf.float32, name="dropout_rate")

            self._initial_state_train = self.cell.zero_state(batch_size, tf.float32)
            self._initial_state_valid = self.cell.zero_state(1, tf.float32)

            # learning rate ops
            self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
            self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * lr_decay)

            # epoch ops
            self.epoch = tf.Variable(0, trainable=False)
            self.epoch_update_op = self.epoch.assign(self.epoch + 1)

            # samples seen ops
            self.samples_seen = tf.Variable(0, trainable=False)
            self.samples_seen_update_op = self.samples_seen.assign(self.samples_seen + batch_size)
            self.samples_seen_reset_op = self.samples_seen.assign(0)

            # global step variable - controled by the model
            self.global_step = tf.Variable(0.0, trainable=False)

            # average loss ops
            self.current_ppx = tf.Variable(1.0, trainable=False)
            self.current_loss = tf.Variable(0.0, trainable=False)
            # self.current_loss_update_op = None

            self.best_eval_ppx = tf.Variable(numpy.inf, trainable=False)
            self.estop_counter = tf.Variable(0, trainable=False)
            self.estop_counter_update_op = self.estop_counter.assign(self.estop_counter + 1)
            self.estop_counter_reset_op = self.estop_counter.assign(0)

            initializer = tf.random_uniform_initializer(minval=init_scale, maxval=init_scale, seed=_SEED)

            out_proj = hidden_size
            if hidden_proj > 0:
                out_proj = hidden_proj

            with tf.device("/cpu:0"):
                w = tf.get_variable("proj_w", [out_proj, vocab_size])
                w_t = tf.transpose(w)
                b = tf.get_variable("proj_b", [vocab_size])
            self.output_projection = (w, b)

            sampled_softmax = False

            # Sampled softmax only makes sense if we sample less than vocabulary size.
            if 0 < num_samples < vocab_size:
                sampled_softmax = True

                def sampled_loss(logits, labels):
                    with tf.device("/cpu:0"):
                        labels = tf.reshape(labels, [-1, 1])
                        losses = tf.nn.sampled_softmax_loss(w_t, b, logits, labels, num_samples, vocab_size)
                        return losses

                loss_function = sampled_loss

            with tf.device("/cpu:0"):
                # input come as one big tensor so we have to split it into a list of tensors to run the rnn cell
                embedding = tf.Variable(
                    tf.random_uniform(
                        [vocab_size, proj_size],
                        minval=-init_scale, maxval=init_scale
                    ),
                    name="embedding"
                )
                # embedding = tf.get_variable("embedding", [vocab_size, proj_size])

                inputs_train = [tf.nn.embedding_lookup(embedding, i) for i in self.input_data_train]
                inputs_valid = [tf.nn.embedding_lookup(embedding, i) for i in self.input_data_valid]

            with tf.variable_scope("RNN", initializer=initializer):

                if attentive:
                    outputs_train, state_train, _ = lm_ops.apply_attentive_lm(
                        self.cell, inputs_train, sequence_length=array_ops.squeeze(math_ops.add_n(self.mask_train)),
                        projection_attention_f=projection_attention_f, output_form=output_form,
                        dropout=self.dropout_feed, initializer=initializer, dtype=tf.float32
                    )

                    outputs_valid, state_valid, _ = lm_ops.apply_attentive_lm(
                        self.cell, inputs_valid, sequence_length=array_ops.squeeze(math_ops.add_n(self.mask_valid)),
                        projection_attention_f=projection_attention_f, output_form=output_form,
                        dropout=self.dropout_feed, initializer=initializer, dtype=tf.float32
                    )

                else:
                   outputs_train, state_train = lm_ops.apply_lm(
                       self.cell, inputs_train, sequence_length=math_ops.add_n(self.mask_train),
                       dropout=self.dropout_feed, dtype=tf.float32
                   )

                   outputs_valid, state_valid = lm_ops.apply_lm(
                       self.cell, inputs_valid, sequence_length=math_ops.add_n(self.mask_valid),
                       dropout=self.dropout_feed, dtype=tf.float32
                   )

                if sampled_softmax is False:
                    logits_train = [tf.nn.xw_plus_b(o, self.output_projection[0], self.output_projection[1])
                                    for o in outputs_train]
                    logits_valid = [tf.nn.xw_plus_b(o, self.output_projection[0], self.output_projection[1])
                                    for o in outputs_valid]
                else:
                    logits_train = outputs_train
                    logits_valid = outputs_valid

            loss_train = seq2seq.sequence_loss_by_example(
                logits_train, self.targets_train, self.mask_train, average_across_timesteps=True
            )

            loss_valid = seq2seq.sequence_loss_by_example(
                logits_valid, self.targets_valid, self.mask_valid, average_across_timesteps=True
            )

            self._cost_train = cost = tf.reduce_sum(loss_train) / float(batch_size)
            self._final_state_train = state_train

            self._cost_valid = tf.reduce_sum(loss_valid) / float(batch_size)
            self._final_state_valid = state_valid

            if not is_training:
                return

            tvars = tf.trainable_variables()
            grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
                                              max_grad_norm)

            opt = optimization_ops.get_optimizer(optimizer, learning_rate)
            self._train_op = opt.apply_gradients(zip(grads, tvars), global_step=self.global_step)
            self._valid_op = tf.no_op()

            self.saver = tf.train.Saver(tf.all_variables())
            self.saver_best = tf.train.Saver(tf.all_variables())