示例#1
0
def run_loop(session,
             modes,
             models,
             summary_ops=None,
             logging=False,
             name_modifier=""):
    """
    Runs models for the specified number of epochs.
    :param session: MonitoredTrainingSession to use
    :param modes: List of modes. Whether to train, evaluate, or test
    :param models: List of models
    :param summary_ops: List of summary operations with one operation per model, or None.
    :param logging: Whether to write summaries for tensorboard.
    :param name_modifier: Suffix to identify the model in tensorboard. Creates a subdirectory.
    :return: List of lists, where each sub-list is the loss/epoch.
    """
    assert (len(modes) == len(models))
    assert (summary_ops is None or len(summary_ops) == len(models))
    assert (summary_ops is None or logging)

    nr_models = len(modes)
    representative_config = models[
        0].config  # used for variables that are assumed to be the same between models

    writers = []
    losses = []
    for i in range(nr_models):
        losses.append([])
        if logging:
            writers.append(
                tf.summary.FileWriter(
                    log_path + os.sep + Mode.to_string(modes[i]).lower() +
                    name_modifier, session.graph))
        else:
            writers.append(None)

    for i in range(representative_config.nr_epochs):
        for m in range(nr_models):
            model = models[m]
            perplexity = run_epoch(
                session,
                model,
                writer=writers[m],
                summary_op=summary_ops[m],
                eval_op=model.train_op if modes[m] == Mode.train else None)
            losses[m].append(perplexity)
            log("%s Epoch: %d Perplexity: %.3f" %
                (Mode.to_string(modes[m]), i + 1, perplexity))

    return losses
示例#2
0
def instantiate_model(mode, config, data_set):
    """
    Instantiates the model with the given configuration and data.
    :param mode: Mode enum specifying train/test/validation.
    :param config:
    :param data_set: a tf.data.dataset
    :return: model, summary operation
    """
    name = Mode.to_string(mode)

    initializer = tf.random_uniform_initializer(-config.init_scale,
                                                config.init_scale)

    with tf.name_scope(name=name):
        model_input = ModelInput(config=config, data=data_set)
        with tf.variable_scope("Model",
                               reuse=tf.AUTO_REUSE,
                               initializer=initializer):
            model = SeqModel(is_training=(mode == Mode.train),
                             config=config,
                             dataset=model_input)

            summary = tf.summary.scalar(name + " Loss", model.normalized_cost)
            if mode == Mode.train:
                summ_lr = tf.summary.scalar("Learning Rate",
                                            model.config.learning_rate)
                summary = tf.summary.merge([summary, summ_lr])
            summ_layers = tf.summary.scalar("Layers", model.config.num_layers)
            summ_bs = tf.summary.scalar("Batch Size", model.config.batch_size)
            summ_grad_norm = tf.summary.scalar("Gradient Norm",
                                               model.config.max_grad_norm)
            summ_keep_prob = tf.summary.scalar("Dropout Keep Probability",
                                               model.config.keep_prob)
            summ_hidden_size = tf.summary.scalar("State Size",
                                                 model.config.hidden_size)
            summary = tf.summary.merge([
                summary, summ_layers, summ_bs, summ_grad_norm, summ_keep_prob,
                summ_hidden_size
            ])

    return model, summary