예제 #1
0
    train_ds.read()
    dev_ds.read()

    test_ds = None
    if args.test is not None:
        test_ds = DataSet(file=args.test, reader=jlr, formatter=formatter)
        test_ds.read()

    train_feats, dev_feats, test_feats = f.load(train_ds, dev_ds, test_ds)
    f.save_vocab(mname)

    input_shape = train_feats[0].shape[1]

    model = SimpleMLP(input_shape,100,3)

    if gpu():
        model.cuda()


    if model_exists(mname) and os.getenv("TRAIN").lower() not in ["y","1","t","yes"]:
        model.load_state_dict(torch.load("models/{0}.model".format(mname)))
    else:
        train(model, train_feats, 500, 1e-2, 90,dev_feats,early_stopping=EarlyStopping(mname))
        torch.save(model.state_dict(), "models/{0}.model".format(mname))


    print_evaluation(model, dev_feats, FEVERLabelSchema())

    if args.test is not None:
        print_evaluation(model, test_feats, FEVERLabelSchema())
예제 #2
0
    waseem_te_composite = CompositeDataset(name="waseem_composite_test")
    for dataset in datasets_te:
        dataset.read()
        waseem_te_composite.add(dataset)



    davidson_te = DataSet(os.path.join("data","davidson.te.csv"),reader=csvreader,formatter=df,name="davidson_test")
    davidson_te.read()

    features = Features(get_feature_functions(mname))
    train_fs, dev_fs, test_fs_primary, test_fs_aux = features.load(waseem_tr_composite, waseem_de_composite, waseem_te_composite, davidson_te)

    print("Number of features in primary: {0}".format(train_fs[0].shape[1]))

    model = MLP(train_fs[0].shape[1],get_model_shape(),3)

    if gpu():
        model.cuda()

    if model_exists(mname) and os.getenv("TRAIN").lower() not in ["y","1","t","yes"]:
        model.load_state_dict(torch.load("models/{0}.model".format(mname)))
    else:
        train(model, train_fs, 50, 1e-3, 45, dev=dev_fs,
              lr_schedule=lambda a, b: exp_lr_scheduler(a, b, 0.5, 5))
        torch.save(model.state_dict(), "models/{0}.model".format(mname))


    create_log_dir(mname)
    print_evaluation(model,test_fs_primary, WaseemLabelSchema(),log="logs/{0}/primary.jsonl".format(mname))
    print_evaluation(model,test_fs_aux, WaseemLabelSchema(),log="logs/{0}/aux.jsonl".format(mname))
예제 #3
0
        }


if __name__ == "__main__":
    bodies = Bodies("data/fnc-1/train_bodies.csv",
                    "data/fnc-1/competition_test_bodies.csv")

    f = Features([FNCTermFrequencyFeatureFunction(bodies)])
    csvr = CSVReader()
    formatter = FNCFormatter(FNCLabelSchema())

    train_ds = DataSet(file="data/fnc-1/train_stances.csv",
                       reader=csvr,
                       formatter=formatter)
    test_ds = DataSet(file="data/fnc-1/competition_test_stances.csv",
                      reader=csvr,
                      formatter=formatter)

    train_ds.read()
    test_ds.read()

    train_feats, _, test_feats = f.load(train_ds, None, test_ds)

    input_shape = train_feats[0].shape[1]
    model = SimpleMLP(input_shape, 100, 4)

    if gpu():
        model.cuda()

    train(model, train_feats, 500, 1e-2, 90, test_feats, clip=5)
예제 #4
0
    print("Number of features: {0}".format(train_fs[0].shape[1]))
    model = MLP(train_fs[0].shape[1], get_model_shape(), 3)

    if gpu():
        model.cuda()

    if model_exists(mname) and os.getenv("TRAIN").lower() not in [
            "y", "1", "t", "yes"
    ]:
        model.load_state_dict(torch.load("models/{0}.model".format(mname)))
    else:
        train(model,
              train_fs,
              50,
              1e-3,
              60,
              dev=dev_fs,
              early_stopping=EarlyStopping(mname),
              lr_schedule=lambda a, b: exp_lr_scheduler(a, b, 0.5, 5))
        torch.save(model.state_dict(), "models/{0}.model".format(mname))

    create_log_dir(mname)
    print_evaluation(model,
                     dev_fs,
                     DavidsonLabelSchema(),
                     log="logs/{0}/dev.jsonl".format(mname))
    print_evaluation(model,
                     test_fs,
                     DavidsonLabelSchema(),
                     log="logs/{0}/test.jsonl".format(mname))
예제 #5
0
                      reader=jlr,
                      formatter=formatter)

    train_ds.read()
    dev_ds.read()
    test_ds.read()

    train_feats, dev_feats, test_feats = f.load(train_ds, dev_ds, test_ds)
    input_shape = train_feats[0].shape[1]

    model = SimpleMLP(input_shape, 100, 3)

    if gpu():
        model.cuda()

    if model_exists(mname):
        model.load_state_dict(torch.load("models/{0}.model".format(mname)))
        final_model = model
    else:
        final_model = train(model,
                            train_feats,
                            500,
                            1e-2,
                            90,
                            dev_feats,
                            early_stopping=EarlyStopping())
        torch.save(model.state_dict(), "models/{0}.model".format(mname))

    print_evaluation(final_model, dev_feats, FEVERLabelSchema())
    print_evaluation(final_model, test_feats, FEVERLabelSchema())