Ejemplo n.º 1
0
def train_model(dataset, transform_type):
    """
    Train specific model on given dataset.
    :param dataset:
    :param transform_type:
    """
    print('Training model ({}) on {}...'.format(transform_type, dataset))
    (X_train, Y_train), (X_test, Y_test) = load_data(dataset)
    nb_examples, img_rows, img_cols, nb_channels = X_train.shape
    nb_classes = Y_train.shape[1]
    input_shape = (img_rows, img_cols, nb_channels)

    X_train = transform(X_train, transform_type)

    model_name = 'model-{}-cnn-{}'.format(dataset, transform_type)
    require_preprocess = False
    if (dataset == DATA.cifar_10):
        require_preprocess = True

    # train
    model = models.create_model(dataset, input_shape, nb_classes)
    models.train(model, X_train, Y_train, model_name, require_preprocess)
    # save to disk
    models.save_model(model, model_name)
    # evaluate the new model
    X_test = transform(X_test, transform_type)
    loaded_model = models.load_model(model_name)
    scores = loaded_model.evaluate(X_test, Y_test, verbose=2)
    print('*** Evaluating the new model: {}'.format(scores))
    del loaded_model
Ejemplo n.º 2
0
def train_composition(dataset, transformation_list):
    """
    Train a model on dataset on which a sequence of transformations applied
    :param dataset: the original dataset
    :param transformation_list: the sequence of transformations
    :return:
    """
    # Apply a sequence of transformations
    (X_train, Y_train), (X_test, Y_test) = load_data(dataset)
    X_train = transform(X_train, transformation_list)

    nb_examples, img_rows, img_cols, nb_channels = X_train.shape
    nb_classes = Y_train.shape[1]
    input_shape = (img_rows, img_cols, nb_channels)

    # Train a model and save
    model_name = 'model-{}-cnn-{}'.format(dataset, 'composition')
    require_preprocess = (dataset == DATA.cifar_10)

    model = models.create_model(dataset, input_shape, nb_classes)
    models.train(model, X_train, Y_train, model_name, require_preprocess)
    # save to disk
    models.save_model(model, model_name)

    # evaluate the new model
    loaded_model = models.load_model(model_name)
    X_test = transform(X_test, transformation_list)

    if require_preprocess:
        X_test = normalize(X_test)

    scores = loaded_model.evaluate(X_test, Y_test, verbose=2)
    print('*** Evaluating the new model: {}'.format(scores))
    del loaded_model
Ejemplo n.º 3
0
def run(task):
    """Run the train/predict flow for `task`."""
    st.markdown(f'<h1 align="center">{task}</h1>', unsafe_allow_html=True)
    train_button = st.sidebar.button("Train")
    sidebar_train_message = st.sidebar.empty()
    main_train_message = st.empty()
    slug = slugify(task)
    trained = is_trained(slug)
    if trained:
        sidebar_train_message.success(TRAINED_MODEL_MESSAGE)
    else:
        sidebar_train_message.warning(NO_TRAINED_MODEL_MESSAGE)
        main_train_message.warning(NO_TRAINED_MODEL_MESSAGE)
    if train_button:
        sidebar_train_message.info(TRAINING_MESSAGE)
        main_train_message.info(TRAINING_MESSAGE)
        train(slug, sidebar_train_message, TRAINING_MESSAGE)
        sidebar_train_message.success(TRAINED_MODEL_MESSAGE)
        main_train_message.empty()
        trained = is_trained(slug)
    if trained:
        show_metrics = st.sidebar.checkbox("Show metrics")
        if show_metrics:
            metrics = get_metrics(slug)
            for key, value in metrics.items():
                st.sidebar.text(f"{key}: {value}")
        user_input = st.text_area("Input")
        st.text("Output")
        if user_input:
            output = predict(slug, user_input)
            display_function = DISPLAY_FUNCTIONS[slug]
            display_function(output)
Ejemplo n.º 4
0
def start():
    print __doc__
    
    regressors = ['Ridge', 'RandomForestRegressor', 'GradientBoostingRegressor'] #! see config.py 
    features = ['user', 'business', 'checkin', 'votes']
    
    models.train(modelnames=regressors, features=features, limit=2000, stemmer_type="RegexpStemmer",
                 standardized=False, predict=True, plot=True)
def main(args):
    data = np.load(args.dataset, mmap_mode='r')
    cut = int(0.9 * len(data))
    idx = np.arange(len(data))
    np.random.shuffle(idx)
    models.train(
        args, theanets.Autoencoder,
        [data[idx[:cut]]], [data[idx[cut:]]],
        monitors={'hid1:out': (0.01, 0.99)})
def main(args):
    data = np.load(args.dataset, mmap_mode='r')
    cut = int(0.9 * len(data))
    models.train(
        args, theanets.recurrent.Autoencoder,
        theanets.recurrent.batches([data[:cut]], args.time_steps, args.batch_size),
        theanets.recurrent.batches([data[cut:]], args.time_steps, args.batch_size),
        monitors={'hid1:out': (0.01, 0.99)},
    )
Ejemplo n.º 7
0
def main():
    train_data = pd.read_csv('data/train.csv')
    bst_data = pd.read_csv('data/bts.csv')

    train_data = shuffle(train_data)
    train, test = train_test_split(train_data, test_size=0.1)
    train = train.dropna()
    test = pd.read_csv('data/LocTest.csv')
    test = test.dropna()

    train_points = train.iloc[:, 1:3].values
    train_path_loss = train.iloc[:, 3:9].values
    bst_points = bst_data.iloc[:, 1:3].values
    location_points = np.concatenate((train_points, bst_points), axis=0)

    latitudes, longitudes = grid.build_location_grid(location_points)
    bts_coordinates = grid.find_bts_coordinates(bst_points, latitudes,
                                                longitudes)
    print("location grid with size [{:d}, {:d}] calculated.".format(
        latitudes.shape[0], longitudes.shape[0]))

    knn = KNeighborsRegressor(n_neighbors=5)
    trained_models = models.train(train_points, train_path_loss, knn)
    fingerprints = fingerprint.get_grids(trained_models, latitudes, longitudes,
                                         bts_coordinates)

    locate.predict_test_locations(fingerprints, latitudes, longitudes, test,
                                  bst_points)
def train_model(data, model_name):
    X, Y = data
    transformation_type = TRANSFORMATION.clean

    model = models.create_model(DATA.CUR_DATASET_NAME)
    print('Training model [{}]...'.format(model_name))
    model = models.train(model, X, Y, model_name)
    print('Saving model...')
    models.save_model(model, model_name)
    print('Done.')

    return model
Ejemplo n.º 9
0
def train_model(data, transformation_type=TRANSFORMATION.clean):
    X, Y = data

    print('Transforming training data set [{}]...'.format(transformation_type))
    X = transform(X, transformation_type)

    model_name = 'model-{}-cnn-{}'.format(DATA.CUR_DATASET_NAME,
                                          transformation_type)
    model = models.create_model(DATA.CUR_DATASET_NAME)
    print('Training model [{}]...'.format(model_name))
    model = models.train(model, X, Y, model_name)
    print('Saving model...')
    models.save_model(model, model_name)
    print('Done.')

    return model
def main(args):
    X = np.load(args.dataset, mmap_mode='r')
    cut = int(0.9 * len(X))

    D = np.load(args.codebook, mmap_mode='r')
    print('D:', D.shape)
    K = sklearn.metrics.pairwise.euclidean_distances(
        X, D, (D * D).sum(axis=1), squared=True).argmin(axis=1)

    def batch(Z):
        def create():
            inputs = np.zeros((args.batch_size, args.time_steps, len(D)), 'f')
            outputs = np.zeros((args.batch_size, args.time_steps), 'i')
            for b in range(args.batch_size):
                o = np.random.randint(len(Z) - args.time_steps - 1)
                inputs[b, :, Z[o:o+args.time_steps]] = 1
                outputs[b] = Z[o+1:o+1+args.time_steps]
            return [inputs, outputs]
        return create

    net = models.train(
        args, theanets.recurrent.Classifier, batch(K), batch(K))
    speak = D[list(net.predict_sequence(K[cut:cut+200], 1000))]
    np.save('/tmp/speak.npy', speak)
Ejemplo n.º 11
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("dataset",
                        type=str,
                        choices=['20_simplified', '6_simplified'],
                        help="Dataset selected.")
    parser.add_argument("pretrained_model",
                        type=str,
                        default="bert-base-uncased",
                        help="Petrained natural language model selected.")
    parser.add_argument(
        "--output_dir",
        type=str,
        default="./saved_models",
        help=
        "The output directory where the results and checkpoints will be written."
    )
    parser.add_argument("--batch_size",
                        type=int,
                        default=64,
                        help="Batch size.")
    parser.add_argument(
        "--max_length",
        type=int,
        default=128,
        help="The maximum total input sequence length after tokenization."
    )  # check for each dataset
    parser.add_argument("--learning_rate",
                        type=float,
                        default=5e-5,
                        help="The initial learning rate for Adam.")
    parser.add_argument("--adam_epsilon",
                        type=float,
                        default=1e-8,
                        help="Epsilon for Adam optimizer.")
    parser.add_argument("--weight_decay",
                        type=float,
                        default=0.01,
                        help="Weight decay if we apply some.")
    parser.add_argument("--num_train_epochs",
                        type=int,
                        default=5,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--num_warmup_steps",
                        type=int,
                        default=0,
                        help="Linear warmup over warmup_steps.")
    parser.add_argument("--max_grad_norm",
                        type=float,
                        default=1.0,
                        help="Max gradient norm.")
    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
    parser.add_argument("--lowercase",
                        action='store_true',
                        help="Uses an uncased model.")
    parser.add_argument("--lstm_hidden_dim",
                        type=int,
                        default=1024,
                        help="Number of LSTM units.")
    parser.add_argument("--dense_hidden_dim",
                        type=int,
                        default=32,
                        help="Number oh dense units.")
    parser.add_argument("--num_lstm_layers",
                        type=int,
                        default=1,
                        help="Number of LSTM layers.")
    parser.add_argument("--num_dense_layers",
                        type=int,
                        default=2,
                        help="Number of dense layers.")
    parser.add_argument("--size_segment",
                        type=int,
                        default=200,
                        help="Size segment.")
    parser.add_argument("--size_shift",
                        type=int,
                        default=50,
                        help="Size shift.")
    parser.add_argument("--lr_lstm",
                        type=float,
                        default=0.001,
                        help="Learning rate of LSTM.")
    parser.add_argument(
        "--epochs_decrease_lr_lstm",
        type=int,
        default=3,
        help=
        "Number of epoch to descrease learning rate when the validation loss does not improve."
    )
    parser.add_argument("--reduced_factor_lstm",
                        type=float,
                        default=0.95,
                        help="Reduced factor for learning rate of the LSTM.")

    args = parser.parse_args()

    dataset2num_labels = {
        '20_simplified': 20,
        '6_simplified': 6,
    }

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    use_token_type_ids = True

    conf_name = (
        f"{args.num_train_epochs}_{args.batch_size}_{args.max_length}_{args.learning_rate}_"
        f"{args.num_warmup_steps}_{args.adam_epsilon}_{args.weight_decay}_{args.max_grad_norm}_{args.seed}_"
        f"{args.lstm_hidden_dim}_{args.dense_hidden_dim}_{args.num_lstm_layers}_{args.num_lstm_layers}"
        f"{args.lr_lstm}_{args.epochs_decrease_lr_lstm}_{args.reduced_factor_lstm}"
    )

    num_classes = dataset2num_labels[args.dataset]

    model_classifier = RoBert(
        num_classes,
        bert_pretrained=args.pretrained_model,
        device=device,
        lstm_hidden_dim=args.lstm_hidden_dim,
        dense_hidden_dim=args.dense_hidden_dim,
        num_lstm_layers=args.num_lstm_layers,
        num_dense_layers=args.num_dense_layers,
    )

    model_tokenizer = model_classifier.tokenizer()

    X_train, y_train, num_segments_train, X_test, y_test, num_segments_test, label2id = load_20newsgroup_segments(
        args.dataset,
        max_length=args.max_length,
        size_segment=args.size_segment,
        size_shift=args.size_shift)

    train_set = generate_dataset_20newsgroup_segments(
        X_train,
        y_train,
        num_segments_train,
        model_tokenizer,
        max_length=args.size_segment)
    validation_set = generate_dataset_20newsgroup_segments(
        X_test,
        y_test,
        num_segments_test,
        model_tokenizer,
        max_length=args.size_segment)

    model_path = os.path.join(args.output_dir, args.dataset,
                              args.pretrained_model, conf_name)

    if os.path.exists(model_path):
        shutil.rmtree(model_path)

    os.makedirs(model_path)

    id2label = {value: key for (key, value) in label2id.items()}

    train(model_classifier,
          train_set,
          validation_set,
          device,
          model_path,
          id2label,
          batch_size=args.batch_size,
          weight_decay=args.weight_decay,
          num_train_epochs=args.num_train_epochs,
          lr=args.learning_rate,
          eps=args.adam_epsilon,
          num_warmup_steps=args.num_warmup_steps,
          max_grad_norm=args.max_grad_norm,
          seed=args.seed,
          use_token_type_ids=use_token_type_ids,
          lr_lstm=args.lr_lstm,
          epochs_decrease_lr_lstm=args.epochs_decrease_lr_lstm,
          reduced_factor_lstm=args.reduced_factor_lstm)
Ejemplo n.º 12
0
model_path = 'tmp/' + 'adagvae' + experiment_name + experiment_id

labels = ["recon_1", "recon_2", "kl_1", "kl_2"]
train_data, test_data = dsprites.get_dsprites(train_size=737280,
                                              test_size=737280,
                                              batch_size=64)
vae = AdaGVAE(n_channels=1)

opt = optim.Adam(vae.parameters(), lr=0.0001, betas=(0.9, 0.999), eps=1e-8)

writer = SummaryWriter(log_dir=model_path)
for epoch in range(29):
    models.train(vae,
                 train_data,
                 epoch,
                 opt,
                 verbose=True,
                 writer=writer,
                 metrics_labels=labels)

_, metrics = models.test(vae,
                         test_data,
                         verbose=True,
                         metrics_labels=labels,
                         writer=writer)

with torch.no_grad():
    num_samples = 15
    x1, x2, _ = next(iter(test_data))
    x1 = x1[:15]
    x2 = x2[:15]
Ejemplo n.º 13
0
            # to wait for torchsummary to be fixed first.

        if CHECKPOINT is not None:
            model.load_state_dict(torch.load(CHECKPOINT))

        if (T_BAND_GROUP != 0):  # KD is used
            try:
                train(
                    model,
                    optimizer,
                    loss,
                    train_loader,
                    hyperparams["epoch"],
                    scheduler=hyperparams["scheduler"],
                    device=hyperparams["device"],
                    supervision=hyperparams["supervision"],
                    val_loader=val_loader,
                    display=viz,
                    name=NAME,
                    t_net=t_model,
                    t_alpha=T_KD_ALPHA,
                    t_temp=T_KD_TEMP,
                    n_bands=N_BANDS,
                )
            except KeyboardInterrupt:
                # Allow the user to stop the training
                pass

            probabilities = test(model, img[:, :, :N_BANDS], hyperparams)
            prediction = np.argmax(probabilities, axis=-1)
Ejemplo n.º 14
0
def test_fc_100_100_10_accuracy(mnist):
    model = fc_100_100_10()
    X_train, y_train, X_test, y_test = mnist

    train(model, X_train, y_train, epochs=2)
    assert isclose(accuracy(model, X_test, y_test), 0.544, abs_tol=0.01)
Ejemplo n.º 15
0
def main(args):
    # Main function flow
    # 0. Load experiment conditions
    config_list, exp_name = _load_params(args.file_path)
    if args.single_mode:
        exp_name = exp_name + "_single"

    # 1. Create Agents
    agents, core_agent = create_agents(config_list)

    # # 2. Create Environments
    envs, core_env = create_envs(agents, core_agent, exp_name)

    # 2.5 Create directory for save temporally
    create_directory(agents, core_agent, exp_name)

    if not args.single_mode:
        # 3. Train model
        # best_agent = models.train(envs, agents, core_env, core_agent, core_agent.CONSTANTS.N_EPISODE, len(agents), exp)
        exp = hyper_dash_settings(exp_name)
        models.train(envs, agents,
                     core_env, core_agent, core_agent.CONSTANTS.N_EPISODE,
                     len(agents), exp, exp_name)
        exp.end()
        # torch.save(best_agent.policy_net, best_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH + "/dqn_pong_model")
        for agent in agents:
            with open(
                    core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
                    "/{}/internal-agent/{}.pkl".format(
                        exp_name, agent.get_name()), 'wb') as f:
                cloudpickle.dump(agent.policy_net, f)
        with open(
                core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
                "/{}/core-agent/{}.pkl".format(
                    exp_name, core_agent.get_name()), 'wb') as f:
            cloudpickle.dump(core_agent.policy_net, f)
    else:
        # 3.Train model
        exp = hyper_dash_settings(exp_name)
        models.single_train(envs, agents, core_env,
                            core_agent, core_agent.CONSTANTS.N_EPISODE,
                            len(agents), exp, exp_name)
        exp.end()
        with open(
                core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
                "/{}/core-agent/{}.pkl".format(
                    exp_name, core_agent.get_name()), 'wb') as f:
            cloudpickle.dump(core_agent.policy_net, f)
    # 4. Test model
    del agents
    test_env = create_test_envs(core_agent, exp_name)
    with open(
            core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
            "/{}/core-agent/{}.pkl".format(exp_name, core_agent.get_name()),
            'rb') as f:
        policy_net = cloudpickle.load(f)
    policy_net.eval()
    exp_test = hyper_dash_settings(exp_name + "_test")
    models.test(test_env,
                1,
                policy_net,
                exp_test,
                exp_name,
                render=False,
                agent=core_agent)
    exp_test.end()
        os.system('pause')
        # ---------------中断程序------------------

        if CHECKPOINT is not None:
            model.load_state_dict(torch.load(CHECKPOINT))

        # 训练模型!!!
        try:
            train(model,
                  optimizer,
                  loss,
                  train_loader,
                  hyperparams['epoch'],
                  scheduler=hyperparams['scheduler'],
                  device=hyperparams['device'],
                  supervision=hyperparams['supervision'],
                  val_loader=val_loader,
                  display=viz,
                  img=img,
                  gt=gt,
                  hyperparams=hyperparams,
                  n_classes=N_CLASSES,
                  viz=viz,
                  LABEL_VALUES=LABEL_VALUES)  # for metric
        except KeyboardInterrupt:
            # Allow the user to stop the training to do inference
            pass

        # 对整个数据集计算了预测,而不仅仅是对test做预测
        probabilities = test(model, img, hyperparams)
        prediction = np.argmax(probabilities, axis=-1)
model = NeuralNetwork(**nn_params)
model = model.to(model.device)
optimizer = optim.Adam(model.parameters(), **opt_params)
loss_fn = torch.nn.CrossEntropyLoss(reduction="elementwise_mean")

# Create callbacks
checkpoint = CheckPoint(model, "modelA.ckpt")
earlystop = EarlyStopping(**earlystop_params)

# Train and evaluate the model
flg_stop = False
for epoch in range(1, params["n_epochs"] + 1):
    print("\n[EPOCH %d]" % (epoch))
    loss_train = train(model,
                       trainA_loader,
                       optimizer,
                       loss_fn,
                       epoch,
                       description="Train on task A")
    print()
    loss_test, acc_test = test(model,
                               testA_loader,
                               loss_fn,
                               description="Test on task A")
    print()

    # Callbacks
    checkpoint.backup(loss_test)
    flg_stop = earlystop.check(loss_test)
    if flg_stop:
        break
    print("------------------------------------------------------------------")
            for input, _ in train_loader:
                break
            summary(model.to(hyperparams['device']), input.size()[1:])

        # ---------------中断程序------------------
        # print([DATASET + ' ' + MODEL])
        os.system('pause')
        # ---------------中断程序------------------

        if CHECKPOINT is not None:
            model.load_state_dict(torch.load(CHECKPOINT))

        # 训练模型!!!
        try:
            train(model, optimizer, loss, train_loader, hyperparams['epoch'],
                  scheduler=hyperparams['scheduler'], device=hyperparams['device'],
                  supervision=hyperparams['supervision'], val_loader=val_loader,
                  display=viz)
        except KeyboardInterrupt:
            # Allow the user to stop the training to do inference
            pass

        # 对整个数据集计算了预测,而不仅仅是对test做预测
        probabilities = test(model, img, hyperparams)
        prediction = np.argmax(probabilities, axis=-1)

        # 2020.4.20增加,多次获取metrics

    # run_results = metrics(prediction, test_gt, ignored_labels=hyperparams['ignored_labels'], n_classes=N_CLASSES)
    run_results = metrics(prediction, gt, ignored_labels=hyperparams['ignored_labels'], n_classes=N_CLASSES)

    mask = np.zeros(gt.shape, dtype='bool')
Ejemplo n.º 19
0
PREFIX = os.environ.get('PREFIX', '.')

X_train, y_train, X_test, y_test = mnist()

if not args.random_seed:
    K.clear_session()
    tf.set_random_seed(1234)
    np.random.seed(1234)

no_defense_model = fc_100_100_10()
print(f"Training {no_defense_model.name}...")
train(no_defense_model,
      X_train,
      y_train,
      args.epochs,
      verbose=True,
      stop_on_stable_weights=True,
      reduce_lr_on_plateau=True,
      stop_on_stable_weights_patience=60,
      reduce_lr_on_plateau_patience=30)

print(f"Saving {no_defense_model.name}...")
save_to_file(no_defense_model, PREFIX)

for n_components in args.n_components:
    if args.pca:
        pca = cached(f"pca-{n_components}")
        filtered_model = pca_filtered_model(no_defense_model,
                                            X_train,
                                            n_components,
                                            pca=pca)
Ejemplo n.º 20
0
args = argument_parser.parse_args()

X_train, y_train, X_test, y_test = mnist()

models_win_counter = {
    "reconstructed_model": 0,
    "retrained_model": 0,
    "reretrained_model": 0,
}

for n_components in [784, 331, 100, 80, 60, 20]:
    reconstructed_model = pca_filtered_model(fc_100_100_10(), X_train,
                                             n_components)
    train(reconstructed_model,
          X_train,
          y_train,
          epochs=-1,
          stop_on_stable_weights=True)

    retrained_model = pca_filtered_model(fc_100_100_10(), X_train,
                                         n_components)
    X_train = retrained_model.preprocessing_fn(X_train)
    train(retrained_model,
          X_train,
          y_train,
          epochs=-1,
          stop_on_stable_weights=True)

    best_model = compare_two_models(reconstructed_model, retrained_model)

    reretrained_model = pca_filtered_model(fc_100_100_10(), X_train,
import models
import preprocess_data
import sys

symbols = sys.argv[1:]  #loads symbols from the command line.
for symbol in symbols:  #runs the models on every symbol.
    df = preprocess_data.loadDataset(symbol)
    scores_models = models.train(df)
    print(
        f"Scores of models on the dataset {symbol} (Linear, Poly, RBF, Sigmoid, RFC, AdaBoost, VotingClassifier): {[score * 100 for score in scores_models]}"
    )
Ejemplo n.º 22
0
# using gpu
parser.add_argument('--gpu',
                    action="store_true",
                    help='add to train on gpu',
                    default=False,
                    dest='train_on_gpu')

# retrieve the results
results = parser.parse_args()

data_dir = results.data_dir
learning_rate = results.learning_rate
epochs = results.epochs
train_on_gpu = results.train_on_gpu
hidden_units = results.hidden_units
arch = results.arch
batch_size = results.batch_size
save_dir = results.save_dir
optim = results.optim

models.train(data_dir,
             lr=learning_rate,
             train_on_gpu=train_on_gpu,
             epochs=epochs,
             hidden_units=hidden_units,
             arch=arch,
             batch_size=batch_size,
             save_dir=save_dir,
             optim=optim)
Ejemplo n.º 23
0
def test_pca_filtered_keeping_10_components_accuracy(mnist):
    X_train, y_train, X_test, y_test = mnist
    model = pca_filtered_model(fc_100_100_10(), X_train, 10)

    train(model, X_train, y_train, epochs=2)
    assert isclose(accuracy(model, X_test, y_test), 0.44, abs_tol=0.01)
Ejemplo n.º 24
0
    def trainBoostedClassifier(self, classifier, level=0):

        milestones = self.config['milestones']
        lr = self.config['lr']

        if self.resume_epoch != -1:

            self.classifiers = self.classifiers[:-1]
            self.weights = self.weights[:-1]

            start = self.resume_epoch

            tmp = -1
            for m in range(len(milestones)):
                if milestones[m] <= self.resume_epoch:
                    lr = lr * self.config['gamma']
                    tmp = m
                else:
                    break

            if tmp != -1:
                milestones = milestones[tmp:]

            milestones = list(np.array(milestones) - self.resume_epoch)
        else:
            start = 0

        id_classifier = len(self.classifiers)

        print(level * "   " + "Training Boosted Classifier n°" +
              str(id_classifier) + "...")

        optimizer = optim.SGD(classifier.parameters(),
                              lr=lr,
                              momentum=self.config['momentum'],
                              weight_decay=self.config['weight_decay'])
        scheduler = lr_scheduler.MultiStepLR(optimizer,
                                             milestones=milestones,
                                             gamma=self.config['gamma'],
                                             last_epoch=-1)

        #Adversarial training for the first classifier
        if id_classifier == 0:

            attack = LinfPGDAttack(classifier,
                                   eps=self.config['eps'] / 255,
                                   eps_iter=self.config['eps_iter'] / 255,
                                   nb_iter=self.config['nb_iter'],
                                   rand_init=self.config['rand_init'],
                                   clip_min=self.config['clip_min'],
                                   clip_max=self.config['clip_max'])

            for epoch in range(start, self.config['epochs']):
                classifier.train()

                models.adversarialTrain(classifier,
                                        self.device,
                                        self.train_loader,
                                        optimizer,
                                        epoch,
                                        attack,
                                        level=level + 1)

                scheduler.step()

                classifier.eval()

                accuracy_under_attack = models.test_under_attack(
                    classifier,
                    self.device,
                    self.test_loader,
                    attack,
                    level=level + 1)
                accuracy = models.test(classifier,
                                       self.device,
                                       self.test_loader,
                                       level=level + 1)

                models.save_model(self.save_dir,
                                  id_classifier,
                                  self.device,
                                  classifier,
                                  accuracy,
                                  accuracy_under_attack,
                                  epoch,
                                  level=level + 1)

                models.updateAndSaveBestAccuracies(self.save_dir,
                                                   id_classifier,
                                                   self.device,
                                                   classifier,
                                                   accuracy,
                                                   accuracy_under_attack,
                                                   level=level + 1)

        else:  #Natural training on the adversarial data set created against the mixture

            adversarial_train_loader, adversarial_test_loader = self.adversarialTrainLoader(
                level=level + 1)

            for epoch in range(start, self.config['epochs']):
                classifier.train()

                models.train(classifier,
                             self.device,
                             adversarial_train_loader,
                             optimizer,
                             epoch,
                             level=level + 1)

                scheduler.step()

                classifier.eval()

                accuracy_under_attack = models.test(classifier,
                                                    self.device,
                                                    adversarial_test_loader,
                                                    level=level + 1)
                accuracy = models.test(classifier,
                                       self.device,
                                       self.test_loader,
                                       level=level + 1)

                models.save_model(self.save_dir,
                                  id_classifier,
                                  self.device,
                                  classifier,
                                  accuracy,
                                  accuracy_under_attack,
                                  epoch,
                                  level=level + 1)

                models.updateAndSaveBestAccuracies(self.save_dir,
                                                   id_classifier,
                                                   self.device,
                                                   classifier,
                                                   accuracy,
                                                   accuracy_under_attack,
                                                   level=level + 1)

        classifier, acc, top_acc_under_attack = models.load_model(
            self.save_dir,
            id_classifier,
            -1,
            self.device,
            self.config['number_of_class'],
            top_acc_under_attack=True)

        self.classifiers.append(classifier)
Ejemplo n.º 25
0
#%%
data, label = load_data(data_path, label_path, 'indian_pines')
#%%
get_value_data(data, label)
#%%
DATA = pd.read_csv('datasets/Indian_pines.csv', header=None).values
data_D = DATA[:, :-1]
data_L = DATA[:, -1]
data_train, data_test, label_train, label_test = train_test_split(
    data_D, data_L, test_size=0.8)
#%%
train_set = GetLoader(data_train, label_train)
train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
val_set = GetLoader(data_test, label_test)
val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False)
#%%
data_p, label_p = next(iter(train_loader))
# print(data_p[:-1])
#%%
net = Baseline(INPUT_CHANNELS, CLASSES, dropout=False)
optimizer = optim.Adam(net.parameters(), lr=0.0001)
weight = torch.ones(CLASSES)
weight[torch.LongTensor([0])] = 0.
w = weight.to(DEVICE)
criterion = nn.CrossEntropyLoss(weight=w)
#%%
train_loss, val_accuracy = train(net, optimizer, criterion, train_loader,
                                 val_loader, EPOCH, DEVICE)

plot_curve(train_loss)
plot_curve(val_accuracy)
Ejemplo n.º 26
0
        "epochs": -1,
        "early_stopping": True,
        "reduce_lr_on_plateau": True,
        "early_stopping_patience": 60
    },
    {
        "epochs": -1,
        "stop_on_stable_weights": True,
        "reduce_lr_on_plateau": True,
        "reduce_lr_on_plateau_patience": 30
    },
]
accuracies_list = []
epochs_list = []

X_train, y_train, X_test, y_test = mnist()

for kwargs in kwargs_list:
    model = fc_100_100_10()
    history = train(model, X_train, y_train, **kwargs)
    n_epochs = len(history.epoch)
    test_set_accuracy = accuracy(model, X_test, y_test)
    accuracies_list.append(test_set_accuracy)
    epochs_list.append(n_epochs)

print("#" * 80)
for kwargs, test_set_accuracy, epochs in zip(kwargs_list, accuracies_list,
                                             epochs_list):
    print(f"{kwargs} -> {test_set_accuracy}, trained for {epochs} epochs")
    print("#" * 80)
Ejemplo n.º 27
0
# to find the best parameters of a given model. A parameters grid should be provided.
if finetune:
    print("Finetuning ...")

    model = models[model_name]['build_fn']
    param_grid = models[model_name]['params']
    gs, fitted_model, pred = search_pipeline(X_train,
                                             X_test,
                                             y_train,
                                             model,
                                             param_grid,
                                             scoring_fit='accuracy')
    best_parameters = get_best_parameters(gs)
    y_pred = m.predict(fitted_model, X_test)
    accuracy(y_test, y_pred > 0.5)
    #results = cross_validation( model, X,y, n_splits=n_sp,  **best_parameters)
if train:
    print("Training ...")
    model = models[model_name]['build_fn']
    history = m.train(model,
                      X_train,
                      y_train,
                      X_test,
                      y_test,
                      batch_size=batch_size,
                      epochs=epochs)
    print("Cross validation ...")
    results = cross_validation(model, X_train, y_train, n_splits=n_sp)
    y_pred = m.predict(history, X_test)
    accuracy(y_test, y_pred > 0.5)
Ejemplo n.º 28
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("dataset",
                        type=str,
                        choices=['20_simplified', '6_simplified'],
                        help="Dataset selected.")
    parser.add_argument("model_type",
                        type=str,
                        choices=['bert', 'distilbert'],
                        help="Petrained natural language model type selected.")
    parser.add_argument(
        "--output_dir",
        type=str,
        default="./saved_models",
        help=
        "The output directory where the results and checkpoints will be written."
    )
    parser.add_argument("--batch_size",
                        type=int,
                        default=64,
                        help="Batch size.")
    parser.add_argument(
        "--max_length",
        type=int,
        default=128,
        help="The maximum total input sequence length after tokenization."
    )  # check for each dataset
    parser.add_argument("--learning_rate",
                        type=float,
                        default=5e-5,
                        help="The initial learning rate for Adam.")
    parser.add_argument("--adam_epsilon",
                        type=float,
                        default=1e-8,
                        help="Epsilon for Adam optimizer.")
    parser.add_argument("--weight_decay",
                        type=float,
                        default=0.01,
                        help="Weight decay if we apply some.")
    parser.add_argument("--num_train_epochs",
                        type=int,
                        default=5,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--num_warmup_steps",
                        type=int,
                        default=0,
                        help="Linear warmup over warmup_steps.")
    parser.add_argument("--max_grad_norm",
                        type=float,
                        default=1.0,
                        help="Max gradient norm.")
    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
    parser.add_argument("--lowercase",
                        action='store_true',
                        help="Uses an uncased model.")

    args = parser.parse_args()

    dataset2num_labels = {
        '20_simplified': 20,
        '6_simplified': 6,
    }

    models_use_token_tipe_ids = frozenset({
        "bert",
    })

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    conf_name = f"{args.num_train_epochs}_{args.batch_size}_{args.max_length}_{args.learning_rate}_{args.num_warmup_steps}_{args.adam_epsilon}_{args.weight_decay}_{args.max_grad_norm}_{args.seed}"
    use_token_type_ids = True if args.model_type in models_use_token_tipe_ids else False
    num_classes = dataset2num_labels[args.dataset]

    model_tokenizer, model_classifier = petrained_language_model_classifier(
        args.model_type,
        dataset2num_labels[args.dataset],
        lowercase=args.lowercase,
    )

    train_set, validation_set, label2id, _, _ = get_dataset(
        model_tokenizer,
        args.dataset,
        max_length=args.max_length,
        seed=args.seed,
        use_token_type_ids=use_token_type_ids,
    )

    model_path = os.path.join(args.output_dir, args.dataset, args.model_type,
                              conf_name)

    if os.path.exists(model_path):
        shutil.rmtree(model_path)

    os.makedirs(model_path)

    id2label = {value: key for (key, value) in label2id.items()}

    train(model_classifier,
          train_set,
          validation_set,
          device,
          model_path,
          id2label,
          batch_size=args.batch_size,
          weight_decay=args.weight_decay,
          num_train_epochs=args.num_train_epochs,
          lr=args.learning_rate,
          eps=args.adam_epsilon,
          num_warmup_steps=args.num_warmup_steps,
          max_grad_norm=args.max_grad_norm,
          seed=args.seed,
          use_token_type_ids=use_token_type_ids)
Ejemplo n.º 29
0
            for input, _ in train_loader:
                break
            summary(model.to(hyperparams["device"]), input.size()[1:])
            # We would like to use device=hyperparams['device'] altough we have
            # to wait for torchsummary to be fixed first.

        if CHECKPOINT is not None:
            model.load_state_dict(torch.load(CHECKPOINT))

        try:
            train(
                model,
                optimizer,
                loss,
                train_loader,
                hyperparams["epoch"],
                scheduler=hyperparams["scheduler"],
                device=hyperparams["device"],
                supervision=hyperparams["supervision"],
                val_loader=val_loader,
                display=viz,
            )
        except KeyboardInterrupt:
            # Allow the user to stop the training
            pass

        probabilities = test(model, img, hyperparams)
        prediction = np.argmax(probabilities, axis=-1)

    run_results = metrics(
        prediction,
        test_gt,
Ejemplo n.º 30
0
def train_and_test(model_name, compression_type, training_size, test_size):
    # train model
    if compression_type == "basic" and test_size == 224:
        start = time.process_time()
        print("training")
        model = train(model_name, train_flow, valid_flow)
        end = time.process_time()
    else:
        start = time.process_time()
        model = Xception(
            weights=None,
            include_top=False,
            #input_shape=(224,224,3)
        )
        model = build_model(model)
        print("loading weight")
        model.load_weights("results/Xception.ckpt")
        end = time.process_time()

    training_time = end - start

    FLOW_MAP = {
        "nearest": test_flow_nearest,
        "box": test_flow_box,
        "lanczos": test_flow_lanczos,
        "hamming": test_flow_hamming,
    }
    test_flow = FLOW_MAP[compression_type]

    # save model
    # np.savez("model_%s.npz" % model_name, **model)

    #Evaluate Model

    # get compression specific test flow
    # test_flow =

    start = time.process_time()
    train_pred = model.predict(valid_flow)
    train_test = valid_flow.classes
    train_accuracy = metrics.accuracy_score(train_test, np.round(train_pred))
    # train_accuracy = history.history["accuracy"]
    test_pred = model.predict(test_flow)
    test_test = test_flow.classes
    test_accuracy = metrics.accuracy_score(test_test, np.round(test_pred))
    print("Sample predictions: ", np.round(test_pred))
    print("Sample actual labels: ", test_test)
    test_matrix = metrics.confusion_matrix(test_test,
                                           np.round(test_pred)).ravel()
    end = time.process_time()

    test_time = end - start

    print("Model: %s\n" % model_name)
    print("Image Compression: %s\n" % compression)
    print("Train Image Size: %d\n" % training_size)
    print("Test Image Size: %d\n" % test_size)
    print("Train accuracy: %f\n" % train_accuracy)
    print("Test accuracy: %f\n" % test_accuracy)
    print("True Negatives: %f\n" % test_matrix[0])
    print("False Positives: %f\n" % test_matrix[1])
    print("False Negatives: %f\n" % test_matrix[2])
    print("True Positives: %f\n" % test_matrix[3])
    print("Training Time: %s sec\n" % training_time)
    print("Testing Time: %s sec\n" % test_time)

    # save results
    try:
        with open("results/%s_results.txt" % model_name, "a") as f:
            f.write("Model: %s\n" % model_name)
            f.write("Image Compression: %s\n" % compression)
            f.write("Train Image Size: %d\n" % training_size)
            f.write("Test Image Size: %d\n" % test_size)
            f.write("Train accuracy: %f\n" % train_accuracy)
            f.write("Test accuracy: %f\n" % test_accuracy)
            f.write("True Negatives: %f\n" % test_matrix[0])
            f.write("False Positives: %f\n" % test_matrix[1])
            f.write("False Negatives: %f\n" % test_matrix[2])
            f.write("True Positives: %f\n" % test_matrix[3])
            f.write("Training Time: %s sec\n" % training_time)
            f.write("Testing Time: %s sec\n" % test_time)
            f.write("\n\n")
            f.close()
    except Exception as e:
        print(e)

    return {
        "Model": model_name,
        "Image Compression": compression,
        "Train Image Size": training_size,
        "Test Image Size": test_size,
        "Training Accuracy": train_accuracy,
        "Test Accuracy": test_accuracy,
        "True Negative": test_matrix[0],
        "False Positive": test_matrix[1],
        "False Negative": test_matrix[2],
        "True Positive": test_matrix[3],
        "Train Time": training_time,
        "Test Time": test_time
    }
Ejemplo n.º 31
0
 export_dir = "/tmp/export/"  
 export_name = "pet-model"  export_version = 2 
 images, labels = datasets.input_pipeline(dataset_dir, batch_size,    is_training=True)
 test_images, test_labels = datasets.input_pipeline(dataset_dir,   batch_size, is_training=False)   
 
 with tf.variable_scope("models") as scope:     
     logits = nets.inference(images, is_training=True)     
     scope.reuse_variables()     
     test_logits = nets.inference(test_images, is_training=False)   
     
     total_loss = models.compute_loss(logits, labels)  
     train_accuracy = models.compute_accuracy(logits, labels)  
     test_accuracy = models.compute_accuracy(test_logits, test_labels)    
     global_step = tf.Variable(0, trainable=False)  
     learning_rate = models.get_learning_rate(global_step,   initial_learning_rate, decay_steps, decay_rate)  
     train_op = models.train(total_loss, learning_rate, global_step,   train_vars)   
     
     saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep)  
     checkpoints_dir = os.path.join(save_dir,   datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))  
     if not os.path.exists(save_dir):     
         os.mkdir(save_dir)  
         if not os.path.exists(checkpoints_dir):     
             os.mkdir(checkpoints_dir)
             
with tf.Session() as sess:     
    sess.run(tf.global_variables_initializer())     
    coords = tf.train.Coordinator()     
    threads = tf.train.start_queue_runners(sess=sess, coord=coords)      
   
    with tf.variable_scope("models"):        
        nets.load_caffe_weights("data/VGG16.npz", sess, ignore_missing=True)      
Ejemplo n.º 32
0
def train(opt):

    # Load data
    loader = DataLoader(opt)
    opt.vocab_size = loader.vocab_size
    opt.seq_length = loader.seq_length

    # Tensorboard summaries (they're great!)
    tb_summary_writer = tb and tb.SummaryWriter(opt.checkpoint_path)

    # Load pretrained model, info file, histories file
    infos = {}
    histories = {}

    if opt.start_from is not None:
        with open(os.path.join(opt.start_from,
                               'infos_' + opt.id + '.pkl')) as f:
            infos = cPickle.load(f)
            saved_model_opt = infos['opt']
            need_be_same = ["rnn_type", "rnn_size", "num_layers"]
            for checkme in need_be_same:
                assert vars(saved_model_opt)[checkme] == vars(
                    opt
                )[checkme], "Command line argument and saved model disagree on '%s' " % checkme
        if os.path.isfile(
                os.path.join(opt.start_from, 'histories_' + opt.id + '.pkl')):
            with open(
                    os.path.join(opt.start_from,
                                 'histories_' + opt.id + '.pkl')) as f:
                histories = cPickle.load(f)
    iteration = infos.get('iter', 0)
    epoch = infos.get('epoch', 0)
    val_result_history = histories.get('val_result_history', {})
    loss_history = histories.get('loss_history', {})
    lr_history = histories.get('lr_history', {})
    #ss_prob_history = histories.get('ss_prob_history', {})
    loader.iterators = infos.get('iterators', loader.iterators)
    loader.split_ix = infos.get('split_ix', loader.split_ix)
    if opt.load_best_score == 1:
        best_val_score = infos.get('best_val_score', None)
    models = Seq2Seq().cuda()
    # Create model

    optimizer = utils.build_optimizer_adam(models.parameters(), opt)

    update_lr_flag = True
    sc_flag = False

    while True:

        # Update learning rate once per epoch
        if update_lr_flag:

            # Assign the learning rate
            if epoch > opt.learning_rate_decay_start and opt.learning_rate_decay_start >= 0:
                frac = (epoch - opt.learning_rate_decay_start
                        ) // opt.learning_rate_decay_every
                decay_factor = opt.learning_rate_decay_rate**frac
                opt.current_lr = opt.learning_rate * decay_factor
            else:
                opt.current_lr = opt.learning_rate
            utils.set_lr(optimizer, opt.current_lr)

            # Assign the scheduled sampling prob
            if epoch > opt.scheduled_sampling_start and opt.scheduled_sampling_start >= 0:
                frac = (epoch - opt.scheduled_sampling_start
                        ) // opt.scheduled_sampling_increase_every
                #opt.ss_prob = min(opt.scheduled_sampling_increase_prob  * frac, opt.scheduled_sampling_max_prob)
                #model.ss_prob = opt.ss_pro

        # Load data from train split (0)
        start = time.time()
        data = loader.get_batch('train')
        data_time = time.time() - start
        start = time.time()

        # Unpack data
        torch.cuda.synchronize()
        tmp = [
            data['fc_feats'], data['att_feats'], data['labels'], data['dist'],
            data['masks'], data['att_masks']
        ]
        tmp = [_ if _ is None else torch.from_numpy(_).cuda() for _ in tmp]
        fc_feats, att_feats, labels, dist_label, masks, attmasks = tmp
        labels = labels.long()
        nd_labels = labels
        batchsize = fc_feats.size(0)
        # Forward pass and loss
        d_steps = 1
        g_steps = 1
        #print (torch.sum(labels!=0), torch.sum(masks!=0))
        if 1:
            if 1:
                models.train()
                optimizer.zero_grad()
                wordact = models(
                    labels.view(batchsize, -1).transpose(1, 0),
                    labels.view(batchsize, -1).transpose(1, 0), fc_feats)
                wordact_t = wordact.transpose(1, 0)[:, 1:, :]
                wordact_t = wordact_t.contiguous().view(
                    wordact_t.size(0) * wordact_t.size(1), -1)
                labels_flat = labels.view(batchsize, -1)
                wordclass_v = labels_flat[:, 1:]
                wordclass_t = wordclass_v.contiguous().view(\
                 wordclass_v.size(0)*wordclass_v.size(1), -1)
                loss_xe = F.cross_entropy(wordact_t[...],
                                          wordclass_t[...].view(-1))

                train_loss = loss_xe
                train_loss.backward()
                optimizer.step()

            if 1:
                if iteration % opt.print_freq == 1:
                    print('Read data:', time.time() - start)
                    if not sc_flag:
                        print("iter {} (epoch {}), train_loss = {:.3f}, data_time = {:.3f}" \
                          .format(iteration, epoch, loss_xe, data_time))
                    else:
                        print("iter {} (epoch {}), avg_reward = {:.3f}, data_time = {:.3f}, time/batch = {:.3f}" \
                            .format(iteration, epoch, np.mean(reward[:,0]), data_time, total_time))

            # Update the iteration and epoch
            iteration += 1
            if data['bounds']['wrapped']:
                epoch += 1
                update_lr_flag = True

            # Write the training loss summary
            if (iteration % opt.losses_log_every == 0):
                add_summary_value(tb_summary_writer, 'train_loss', train_loss,
                                  iteration)
                add_summary_value(tb_summary_writer, 'learning_rate',
                                  opt.current_lr, iteration)
                #add_summary_value(tb_summary_writer, 'scheduled_sampling_prob', model.ss_prob, iteration)
                if sc_flag:
                    add_summary_value(tb_summary_writer, 'avg_reward',
                                      np.mean(reward[:, 0]), iteration)
                loss_history[
                    iteration] = train_loss if not sc_flag else np.mean(
                        reward[:, 0])
                lr_history[iteration] = opt.current_lr
                #ss_prob_history[iteration] = model.ss_prob

        # Validate and save model
            if (iteration % opt.save_checkpoint_every == 0):
                checkpoint_path = os.path.join(
                    opt.checkpoint_path,
                    'langmodel{:05d}.pth'.format(iteration))
                torch.save(models.state_dict(), checkpoint_path)

                optimizer_path = os.path.join(opt.checkpoint_path,
                                              'optimizer.pth')
                torch.save(optimizer.state_dict(), optimizer_path)
Ejemplo n.º 33
0
    queries, tokenized_queries, query_terms_inverted, query_term_ids = create_query_resources(
    )

    inverted_index, tf_C, query_word_positions, unique_terms_per_document, avg_doc_length, document_length, \
        collection_length = build_misc_resources(document_ids, query_terms_inverted)
    document_term_freqs = inverted_index

    create_all_lexical_run_files(index,
                                 document_ids,
                                 queries,
                                 document_term_freqs,
                                 collection_length,
                                 tf_C,
                                 tokenized_queries,
                                 background_model=tf_C,
                                 idf2df=id2df,
                                 num_documents=num_documents)
    features = extract_features(queries, document_ids, index,\
            document_term_freqs, avg_doc_length, id2df, num_documents, tokenized_queries, collection_length)

    datasets, test_features = zip(
        *[get_dataset_for_features(features, i=i) for i in range(10)])
    test_features = test_features[0]

    id2ext, ext2id = get_document_id_maps(index, document_ids)

    for i, dataset in enumerate(datasets):
        model = train(dataset)
        evaluate_model(model, test_features, "./regression", i)
Ejemplo n.º 34
0
        train_data = tenseLoader('train')
        train_len = train_data.len()

        params.KL_W = 0.0
        total_bleu = 0.0
        total_loss = 0.0
        total_KL = 0.0
        bleu_400 = 0.0
        loss_400 = 0.0
        KL_400 = 0.0
        random_list = np.random.permutation(train_len)
        for train_i in range(1, train_len + 1):  #, train_len+1
            input_tensor, input_cond, target_tensor, target_cond = train_data.getPair(
                random_list[train_i - 1])
            input, output, target, use_TF, bleu4, loss, KLloss = models.train(
                input_tensor, target_tensor, input_cond, target_cond, encoder,
                decoder, cond_embed, encoder_optimizer, decoder_optimizer,
                cond_embed_optimizer)
            models.KL_anneal(train_i)

            total_bleu += bleu4
            total_loss += loss
            total_KL += KLloss
            bleu_400 += bleu4
            loss_400 += loss
            KL_400 += KLloss
            file_bleu.write('%f\n' % (bleu4))
            file_loss.write('%f\n' % (loss))
            file_KL.write('%f\n' % (KLloss))

            if (train_i % 400 == 0):
                print('vocab %d/%d: %s/%s(%s), bleu=%4f, loss=%4f, KL=%4f' %