def main():
print("Extraction")
lTrain = futils.open_json("../dataset/my_datasets/train.json")
lDev = futils.open_json("../dataset/my_datasets/dev.json")
tweets_train = get_label_col(0, lTrain)
tweets_dev = get_label_col(0, lDev)
sources_train = get_label_col(1, lTrain)
sources_dev = get_label_col(1, lDev)
objType_train = create_objType(lTrain)
objType_Dev = create_objType(lDev)
print("Preprocessing")
lTrain = hyp3_pp(lTrain)
lDev = hyp3_pp(lDev)
print("Vectorisation")
lTrain, all_words = vutils.vectorise_train(lTrain)
lDev = vutils.vectorise_test(lDev, all_words)
lTrain = add_objType_to_vec(lTrain, objType_train)
lDev = add_objType_to_vec(lDev, objType_Dev)
lTrain = add_source_to_vecs(tweets_train, sources_train, lTrain)
lDev = add_source_to_vecs(tweets_dev, sources_dev, lDev)
print("Saving")
futils.create_json("../dataset/my_datasets/hyp3_train_vecs.json",
lTrain)
futils.create_json("../dataset/my_datasets/hyp3_dev_vecs.json",
lDev)
def main():
print("Extraction")
lTrain = futils.open_json("../dataset/my_datasets/train.json")
lDev = futils.open_json("../dataset/my_datasets/dev.json")
print("Preprocessing")
lTrain = get_label_col(3, lTrain)
lDdev = get_label_col(3, lDev)
print("Vectorisation")
lTrain, all_words = vutils.vectorise_train(lTrain)
lDdev = vutils.vectorise_test(lDev, all_words)
print("Saving")
futils.create_json("../dataset/my_datasets/hyp4_train_vecs.json", lTrain)
futils.create_json("../dataset/my_datasets/hyp4_dev_vecs.json", lDdev)
def main():
print("Extraction")
lTrain = futils.open_json("../dataset/my_datasets/train.json")
lDev = futils.open_json("../dataset/my_datasets/dev.json")
objType_train = create_objType(lTrain)
objType_Dev = create_objType(lDev)
print("Preprocessing")
lTrain = hyp2_pp(lTrain)
lDev = hyp2_pp(lDev)
print("Vectorisation")
lTrain, all_words = vutils.vectorise_train(lTrain)
lDev = vutils.vectorise_test(lDev, all_words)
lTrain = add_objType_to_vec(lTrain, objType_train)
lDev = add_objType_to_vec(lDev, objType_Dev)
print("Saving")
futils.create_json("../dataset/my_datasets/hyp2_train_vecs.json", lTrain)
futils.create_json("../dataset/my_datasets/hyp2_dev_vecs.json", lDev)