def xahoi_thethao_mix(): print("xahoi_thethao_mix") train_files = ["../data_conll_topic/News_Train/Xa_hoi.muc", "../data_conll_topic/News_Train/The_thao.muc"] dev_files = ["../data_conll_topic/News_Dev/The_thao.muc"] test_files = ["../data_conll_topic/News_Test/The_thao.muc"] ner.run(train_files, dev_files, test_files, 500)
def kinhte_thethao_mix_no_shuffle(): print("kinhte_thethao_mix_no_shuffle") train_files = ["../data_conll_topic/News_Train/Kinh_te.muc", "../data_conll_topic/News_Train/The_thao.muc"] dev_files = ["../data_conll_topic/News_Dev/The_thao.muc"] test_files = ["../data_conll_topic/News_Test/The_thao.muc"] ner.run(train_files, dev_files, test_files, 500)
def xahoi_thethao_transfer(w): print("xahoi_thethao_transfer") train_files = [("../data_conll_topic/News_Train/Xa_hoi.muc", 0), ("../data_conll_topic/News_Train/The_thao.muc", 1)] dev_files = [("../data_conll_topic/News_Dev/The_thao.muc", 1)] test_files = [("../data_conll_topic/News_Test/The_thao.muc", 1)] ner.run(train_files, dev_files, test_files, 500, weigth = w)
def kinhte_vanhoa_transfer_weigth(w): print("kinhte_vanhoa_transfer_no_shuffle") train_files = [("../data_conll_topic/News_Train/Kinh_te.muc", 0), ("../data_conll_topic/News_Train/Van_hoa.muc", 1)] dev_files = [("../data_conll_topic/News_Dev/Van_hoa.muc", 1)] test_files = [("../data_conll_topic/News_Test/Van_hoa.muc", 1)] ner.run(train_files, dev_files, test_files, 500, weigth = w)
def kinhte_thethao_transfer_no_shuffle2(): print("kinhte_thethao_transfer_no_shuffle2") train_files = [("../data_conll_topic/News_Train/The_thao.muc", 0), ("../data_conll_topic/News_Train/Kinh_te.muc", 1)] dev_files = [("../data_conll_topic/News_Dev/The_thao.muc", 1)] test_files = [("../data_conll_topic/News_Test/The_thao.muc", 1)] ner.run(train_files, dev_files, test_files, 500)
def kinhte_vanhoa_mix_no_shuffle(): print("kinhte_vanhoa_mix_no_shuffle") train_files = ["../data_conll_topic/News_Train/Kinh_te.muc", "../data_conll_topic/News_Train/Van_hoa.muc"] dev_files = ["../data_conll_topic/News_Dev/Van_hoa.muc"] test_files = ["../data_conll_topic/News_Test/Van_hoa.muc"] ner.run(train_files, dev_files, test_files, 500)
def thethao_kinhte_mix(): train_files = ["../folds/fold_1/Kinh_te.train", "../folds/fold_1/The_thao.train", "../folds/fold_1/The_thao.dev", "../folds/fold_1/The_thao.test"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)
def kinhte_thethao_transfer(): train_files = [("../folds/fold_1/The_thao.train", 1), ("../folds/fold_1/Kinh_te.train", 0), ("../folds/fold_1/Kinh_te.dev", 0), ("../folds/fold_1/Kinh_te.test", 0)] dev_files = [("../folds/fold_1/The_thao.dev", 1)] test_files = [("../folds/fold_1/The_thao.test", 1)] ner.run(train_files, dev_files, test_files, 500)
def t1(): # old topics = ["Doi_song","Giao_duc","Kinh_te","The_gioi",\ "Van_hoa","Giai_tri","KH-CN","Phap_luat","The_thao","Xa_hoi"] train_files = [] test_files = [] for topic in topics: train_files.append("../data_conll_topic/Train/%s.muc" % topic) test_files.append("../data_conll_topic/Dev/%s.muc" % topic) ner.run(train_files, test_files, 50)
def all_no_shuffle(): print("all_no_shuffle") train_files = [] dev_files = [] test_files = [] topics = ["Doi_song","Giao_duc","Kinh_te","The_gioi",\ "Van_hoa","Giai_tri","KH-CN","Phap_luat","The_thao","Xa_hoi"] for topic in topics: train_files.append("../data_conll_topic/News_Train/%s.muc" % topic) dev_files.append("../data_conll_topic/News_Dev/%s.muc" % topic) test_files.append("../data_conll_topic/News_Test/%s.muc" % topic) ner.run(train_files, dev_files, test_files, 500)
def social_1(): # topics = ["Cong_nghe","Giao_duc","Nha_dat","The_thao","Doi_song","Khoa_hoc","Phap_luat","Van_hoa","Giai_tri","Kinh_te","The_gioi","Xa_hoi"] topics = ["Cong_nghe","Giao_duc","Nha_dat","The_thao","Doi_song","Khoa_hoc","Phap_luat"] # topics = ["Cong_nghe", "Kinh_te"] train_files = [] dev_files = [] test_files = [] for topic in topics: train_files.append("../data_conll_topic/Social_Train/%s.muc" % topic) dev_files.append("../data_conll_topic/Social_Dev/%s.muc" % topic) test_files.append("../data_conll_topic/Social_Test/%s.muc" % topic) ner.run(train_files, dev_files, test_files, 100)
def news_1_no_val(): # topics = ["Doi_song","Giao_duc","Kinh_te","The_gioi",\ # "Van_hoa","Giai_tri","KH-CN","Phap_luat","The_thao","Xa_hoi"] topics = ["KH-CN", "Kinh_te"] train_files = [] dev_files = [] test_files = [] for topic in topics: train_files.append("../data_conll_topic/News_Train/%s.muc" % topic) dev_files.append("../data_conll_topic/News_Dev/%s.muc" % topic) test_files.append("../data_conll_topic/News_Test/%s.muc" % topic) ner.run(train_files, dev_files, test_files, 50, no_val=True)
def kinh_te_2_lstm(): train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)
def kinhte_transfer(src): train_files = [("../data_conll_topic/News_Train/%s.muc" % src, 0), ("../data_conll_topic/News_Train/Kinh_te.muc", 1)] dev_files = [("../data_conll_topic/News_Dev/Kinh_te.muc", 1)] test_files = [("../data_conll_topic/News_Test/Kinh_te.muc", 1)] ner.run(train_files, dev_files, test_files, 500, name="%s_kinhte_w_0.3", weigth = 0.3)
def kinhte_mix(src): train_files = ["../data_conll_topic/News_Train/%s.muc" % src, "../data_conll_topic/News_Train/Kinh_te.muc"] dev_files = ["../data_conll_topic/News_Dev/Kinh_te.muc"] test_files = ["../data_conll_topic/News_Test/Kinh_te.muc"] ner.run(train_files, dev_files, test_files, 500, name="%s_kinhte_mix")
def vanhoa_kinhte_indomain(in_domain): train_files = [ "../data_conll_topic/News_Train/Kinh_te.muc"] dev_files = ["../data_conll_topic/News_Dev/Kinh_te.muc"] test_files = ["../data_conll_topic/News_Test/Kinh_te.muc"] ner.run(train_files, dev_files, test_files, 500, in_domain=in_domain)
def vanhoa_kinhte_transfer(): train_files = [("../data_conll_topic/News_Train/Van_hoa.muc", 0), ("../data_conll_topic/News_Train/Kinh_te.muc", 1)] dev_files = [("../data_conll_topic/News_Dev/Kinh_te.muc", 1)] test_files = [("../data_conll_topic/News_Test/Kinh_te.muc", 1)] ner.run(train_files, dev_files, test_files, 500, name="vanhoa_kinhte_w_0", weigth = 0.0)
def kinh_te_transfer_model(): train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)
def topic_2_lstm(topic): train_files = ["../folds/fold_1/%s.train" % topic] dev_files = ["../folds/fold_1/%s.dev" % topic] test_files = ["../folds/fold_1/%s.test" % topic] ner.run(train_files, dev_files, test_files, 500, name=topic)
def kinhte_thethao_outdomain(): train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/The_thao.dev"] test_files = ["../folds/fold_1/The_thao.test"] ner.run(train_files, dev_files, test_files, 500)
def thethao_xahoi_out(): print("thethao_xahoi_out") train_files = ["../data_conll_topic/News_Train/The_thao.muc"] dev_files = ["../data_conll_topic/News_Dev/Xa_hoi.muc"] test_files = ["../data_conll_topic/News_Test/Xa_hoi.muc"] ner.run(train_files, dev_files, test_files, 500)
def vanhoa_kinhte_outdomain(): print("vanhoa_kinhte_outdomain") train_files = ["../data_conll_topic/News_Train/Van_hoa.muc"] dev_files = ["../data_conll_topic/News_Dev/Kinh_te.muc"] test_files = ["../data_conll_topic/News_Test/Kinh_te.muc"] ner.run(train_files, dev_files, test_files, 500)
def vanhoa_kinhte_transfer_no_shuffle(): train_files = [("../data_conll_topic/News_Train/Van_hoa.muc", 0), ("../data_conll_topic/News_Train/Kinh_te.muc", 1)] dev_files = [("../data_conll_topic/News_Dev/Kinh_te.muc", 1)] test_files = [("../data_conll_topic/News_Test/Kinh_te.muc", 2)] ner.run(train_files, dev_files, test_files, 500)
def thethao_2_lstm(): print("thethao_2_lstm") train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)