Esempio n. 1
0
def test_it(hyp):
    for_graph = []
    str0 = ''
    print(hyp)
    str0 += hyp + '\n'
    print("Extraction")
    train_vecs = np.array(
        futils.open_json(
            "../dataset/my_datasets/{}_train_vecs.json".format(hyp)))
    train_labels = np.array(
        futils.open_json("../dataset/my_datasets/train_label.json"))
    test_vecs = np.array(
        futils.open_json(
            "../dataset/my_datasets/{}_dev_vecs.json".format(hyp)))
    test_labels = np.array(
        futils.open_json("../dataset/my_datasets/dev_label.json"))
    print("Training")
    model = nn.create_trained_nn(train_vecs, train_labels, 1)
    epochs = 1
    loss, acc = model.evaluate(train_vecs, train_labels)
    while acc < 0.94 and epochs < 300:
        for_graph.append((epochs, loss, acc))
        loss, acc = model.evaluate(train_vecs, train_labels)
        epochs += 1
        print("Epochs: {}".format(epochs))
        nn.retrain_model(model, train_vecs, train_labels, 1)
    pred_label = nn.predict_nn(model, test_vecs, test_labels)
    pred_label = [(vc.convert_map_to_label(p), vc.convert_map_to_label(l))
                  for p, l in pred_label]
    str0 += ms.all_mesure(pred_label)
    str0 += "Epochs: {}".format(epochs)
    futils.create_json("../tests/for_graph/{}.json".format(hyp), for_graph)
    with open("../tests/nn_{}.txt".format(hyp), 'w+') as stream:
        stream.write(str0)
def main():
    print("Extraction")
    lTrain = futils.open_json("../dataset/my_datasets/train.json")
    lDev = futils.open_json("../dataset/my_datasets/dev.json")
    tweets_train = get_label_col(0, lTrain)
    tweets_dev = get_label_col(0, lDev)
    sources_train = get_label_col(1, lTrain)
    sources_dev = get_label_col(1, lDev)
    objType_train = create_objType(lTrain)
    objType_Dev = create_objType(lDev)
    print("Preprocessing")
    lTrain = hyp3_pp(lTrain)
    lDev = hyp3_pp(lDev)
    print("Vectorisation")
    lTrain, all_words = vutils.vectorise_train(lTrain)
    lDev = vutils.vectorise_test(lDev, all_words)
    lTrain = add_objType_to_vec(lTrain, objType_train)
    lDev = add_objType_to_vec(lDev, objType_Dev)
    lTrain = add_source_to_vecs(tweets_train, sources_train, lTrain)
    lDev = add_source_to_vecs(tweets_dev, sources_dev, lDev)

    print("Saving")
    futils.create_json("../dataset/my_datasets/hyp3_train_vecs.json",
                       lTrain)
    futils.create_json("../dataset/my_datasets/hyp3_dev_vecs.json",
                       lDev)
Esempio n. 3
0
def main():
    lTrain = futils.open_json("../dataset/my_datasets/train.json")
    lDev = futils.open_json("../dataset/my_datasets/dev.json")

    futils.create_json("../dataset/my_datasets/train_label.json",
                       get_label_col(2, lTrain))
    futils.create_json("../dataset/my_datasets/dev_label.json",
                       get_label_col(2, lDev))
def main():
    # aa = futils.open_json(
    #    "../dataset/rumoureval-data/charliehebdo/552783667052167168/structure.json")
    # a = {1: {3: {5: [], 7: []}, 2: {4: {6: {8: []}}}}}
    # b = {}
    # recurs_struc(a, b)
    # print(b)
    futils.create_json(
        "../dataset/traindev/direct_structures.json", direct_struc())
Esempio n. 5
0
def create_ds_json(path, ds):
    dic_ds = {
        rp.get_id():
        [rp.get_id(),
         rp.get_source(),
         rp.get_categorie(),
         rp.get_text()]
        for rp in ds
    }
    list_it = [dic_ds[rp_id] for rp_id in sorted(dic_ds)]
    futils.create_json(path, list_it)
Esempio n. 6
0
def main():
    print("Extraction")
    lTrain = futils.open_json("../dataset/my_datasets/train.json")
    lDev = futils.open_json("../dataset/my_datasets/dev.json")
    print("Preprocessing")
    lTrain = get_label_col(3, lTrain)
    lDdev = get_label_col(3, lDev)
    print("Vectorisation")
    lTrain, all_words = vutils.vectorise_train(lTrain)
    lDdev = vutils.vectorise_test(lDev, all_words)
    print("Saving")
    futils.create_json("../dataset/my_datasets/hyp4_train_vecs.json", lTrain)
    futils.create_json("../dataset/my_datasets/hyp4_dev_vecs.json", lDdev)
Esempio n. 7
0
def main():
    print("Extraction")
    lTrain = futils.open_json("../dataset/my_datasets/train.json")
    lDev = futils.open_json("../dataset/my_datasets/dev.json")
    objType_train = create_objType(lTrain)
    objType_Dev = create_objType(lDev)
    print("Preprocessing")
    lTrain = hyp2_pp(lTrain)
    lDev = hyp2_pp(lDev)
    print("Vectorisation")
    lTrain, all_words = vutils.vectorise_train(lTrain)
    lDev = vutils.vectorise_test(lDev, all_words)
    lTrain = add_objType_to_vec(lTrain, objType_train)
    lDev = add_objType_to_vec(lDev, objType_Dev)
    print("Saving")
    futils.create_json("../dataset/my_datasets/hyp2_train_vecs.json", lTrain)
    futils.create_json("../dataset/my_datasets/hyp2_dev_vecs.json", lDev)