def xahoi_thethao_mix():
print("xahoi_thethao_mix")
train_files = ["../data_conll_topic/News_Train/Xa_hoi.muc",
"../data_conll_topic/News_Train/The_thao.muc"]
dev_files = ["../data_conll_topic/News_Dev/The_thao.muc"]
test_files = ["../data_conll_topic/News_Test/The_thao.muc"]
ner.run(train_files, dev_files, test_files, 500)
def kinhte_thethao_mix_no_shuffle():
print("kinhte_thethao_mix_no_shuffle")
train_files = ["../data_conll_topic/News_Train/Kinh_te.muc",
"../data_conll_topic/News_Train/The_thao.muc"]
dev_files = ["../data_conll_topic/News_Dev/The_thao.muc"]
test_files = ["../data_conll_topic/News_Test/The_thao.muc"]
ner.run(train_files, dev_files, test_files, 500)
def xahoi_thethao_transfer(w):
print("xahoi_thethao_transfer")
train_files = [("../data_conll_topic/News_Train/Xa_hoi.muc", 0),
("../data_conll_topic/News_Train/The_thao.muc", 1)]
dev_files = [("../data_conll_topic/News_Dev/The_thao.muc", 1)]
test_files = [("../data_conll_topic/News_Test/The_thao.muc", 1)]
ner.run(train_files, dev_files, test_files, 500, weigth = w)
def kinhte_vanhoa_transfer_weigth(w):
print("kinhte_vanhoa_transfer_no_shuffle")
train_files = [("../data_conll_topic/News_Train/Kinh_te.muc", 0),
("../data_conll_topic/News_Train/Van_hoa.muc", 1)]
dev_files = [("../data_conll_topic/News_Dev/Van_hoa.muc", 1)]
test_files = [("../data_conll_topic/News_Test/Van_hoa.muc", 1)]
ner.run(train_files, dev_files, test_files, 500, weigth = w)
def kinhte_thethao_transfer_no_shuffle2():
print("kinhte_thethao_transfer_no_shuffle2")
train_files = [("../data_conll_topic/News_Train/The_thao.muc", 0),
("../data_conll_topic/News_Train/Kinh_te.muc", 1)]
dev_files = [("../data_conll_topic/News_Dev/The_thao.muc", 1)]
test_files = [("../data_conll_topic/News_Test/The_thao.muc", 1)]
ner.run(train_files, dev_files, test_files, 500)
def kinhte_vanhoa_mix_no_shuffle():
print("kinhte_vanhoa_mix_no_shuffle")
train_files = ["../data_conll_topic/News_Train/Kinh_te.muc",
"../data_conll_topic/News_Train/Van_hoa.muc"]
dev_files = ["../data_conll_topic/News_Dev/Van_hoa.muc"]
test_files = ["../data_conll_topic/News_Test/Van_hoa.muc"]
ner.run(train_files, dev_files, test_files, 500)
def thethao_kinhte_mix(): train_files = ["../folds/fold_1/Kinh_te.train", "../folds/fold_1/The_thao.train", "../folds/fold_1/The_thao.dev", "../folds/fold_1/The_thao.test"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)
def kinhte_thethao_transfer():
train_files = [("../folds/fold_1/The_thao.train", 1),
("../folds/fold_1/Kinh_te.train", 0),
("../folds/fold_1/Kinh_te.dev", 0),
("../folds/fold_1/Kinh_te.test", 0)]
dev_files = [("../folds/fold_1/The_thao.dev", 1)]
test_files = [("../folds/fold_1/The_thao.test", 1)]
ner.run(train_files, dev_files, test_files, 500)
def t1(): # old
topics = ["Doi_song","Giao_duc","Kinh_te","The_gioi",\
"Van_hoa","Giai_tri","KH-CN","Phap_luat","The_thao","Xa_hoi"]
train_files = []
test_files = []
for topic in topics:
train_files.append("../data_conll_topic/Train/%s.muc" % topic)
test_files.append("../data_conll_topic/Dev/%s.muc" % topic)
ner.run(train_files, test_files, 50)
def all_no_shuffle():
print("all_no_shuffle")
train_files = []
dev_files = []
test_files = []
topics = ["Doi_song","Giao_duc","Kinh_te","The_gioi",\
"Van_hoa","Giai_tri","KH-CN","Phap_luat","The_thao","Xa_hoi"]
for topic in topics:
train_files.append("../data_conll_topic/News_Train/%s.muc" % topic)
dev_files.append("../data_conll_topic/News_Dev/%s.muc" % topic)
test_files.append("../data_conll_topic/News_Test/%s.muc" % topic)
ner.run(train_files, dev_files, test_files, 500)
def social_1():
# topics = ["Cong_nghe","Giao_duc","Nha_dat","The_thao","Doi_song","Khoa_hoc","Phap_luat","Van_hoa","Giai_tri","Kinh_te","The_gioi","Xa_hoi"]
topics = ["Cong_nghe","Giao_duc","Nha_dat","The_thao","Doi_song","Khoa_hoc","Phap_luat"]
# topics = ["Cong_nghe", "Kinh_te"]
train_files = []
dev_files = []
test_files = []
for topic in topics:
train_files.append("../data_conll_topic/Social_Train/%s.muc" % topic)
dev_files.append("../data_conll_topic/Social_Dev/%s.muc" % topic)
test_files.append("../data_conll_topic/Social_Test/%s.muc" % topic)
ner.run(train_files, dev_files, test_files, 100)
def news_1_no_val():
# topics = ["Doi_song","Giao_duc","Kinh_te","The_gioi",\
# "Van_hoa","Giai_tri","KH-CN","Phap_luat","The_thao","Xa_hoi"]
topics = ["KH-CN", "Kinh_te"]
train_files = []
dev_files = []
test_files = []
for topic in topics:
train_files.append("../data_conll_topic/News_Train/%s.muc" % topic)
dev_files.append("../data_conll_topic/News_Dev/%s.muc" % topic)
test_files.append("../data_conll_topic/News_Test/%s.muc" % topic)
ner.run(train_files, dev_files, test_files, 50, no_val=True)
def kinh_te_2_lstm(): train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)
def kinhte_transfer(src):
train_files = [("../data_conll_topic/News_Train/%s.muc" % src, 0),
("../data_conll_topic/News_Train/Kinh_te.muc", 1)]
dev_files = [("../data_conll_topic/News_Dev/Kinh_te.muc", 1)]
test_files = [("../data_conll_topic/News_Test/Kinh_te.muc", 1)]
ner.run(train_files, dev_files, test_files, 500, name="%s_kinhte_w_0.3", weigth = 0.3)
def kinhte_mix(src): train_files = ["../data_conll_topic/News_Train/%s.muc" % src, "../data_conll_topic/News_Train/Kinh_te.muc"] dev_files = ["../data_conll_topic/News_Dev/Kinh_te.muc"] test_files = ["../data_conll_topic/News_Test/Kinh_te.muc"] ner.run(train_files, dev_files, test_files, 500, name="%s_kinhte_mix")
def vanhoa_kinhte_indomain(in_domain): train_files = [ "../data_conll_topic/News_Train/Kinh_te.muc"] dev_files = ["../data_conll_topic/News_Dev/Kinh_te.muc"] test_files = ["../data_conll_topic/News_Test/Kinh_te.muc"] ner.run(train_files, dev_files, test_files, 500, in_domain=in_domain)
def vanhoa_kinhte_transfer():
train_files = [("../data_conll_topic/News_Train/Van_hoa.muc", 0),
("../data_conll_topic/News_Train/Kinh_te.muc", 1)]
dev_files = [("../data_conll_topic/News_Dev/Kinh_te.muc", 1)]
test_files = [("../data_conll_topic/News_Test/Kinh_te.muc", 1)]
ner.run(train_files, dev_files, test_files, 500, name="vanhoa_kinhte_w_0", weigth = 0.0)
def kinh_te_transfer_model(): train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/Kinh_te.dev"] test_files = ["../folds/fold_1/Kinh_te.test"] ner.run(train_files, dev_files, test_files, 500)
def topic_2_lstm(topic): train_files = ["../folds/fold_1/%s.train" % topic] dev_files = ["../folds/fold_1/%s.dev" % topic] test_files = ["../folds/fold_1/%s.test" % topic] ner.run(train_files, dev_files, test_files, 500, name=topic)
def kinhte_thethao_outdomain(): train_files = ["../folds/fold_1/Kinh_te.train"] dev_files = ["../folds/fold_1/The_thao.dev"] test_files = ["../folds/fold_1/The_thao.test"] ner.run(train_files, dev_files, test_files, 500)
def thethao_xahoi_out():
print("thethao_xahoi_out")
train_files = ["../data_conll_topic/News_Train/The_thao.muc"]
dev_files = ["../data_conll_topic/News_Dev/Xa_hoi.muc"]
test_files = ["../data_conll_topic/News_Test/Xa_hoi.muc"]
ner.run(train_files, dev_files, test_files, 500)
def vanhoa_kinhte_outdomain():
print("vanhoa_kinhte_outdomain")
train_files = ["../data_conll_topic/News_Train/Van_hoa.muc"]
dev_files = ["../data_conll_topic/News_Dev/Kinh_te.muc"]
test_files = ["../data_conll_topic/News_Test/Kinh_te.muc"]
ner.run(train_files, dev_files, test_files, 500)
def vanhoa_kinhte_transfer_no_shuffle():
train_files = [("../data_conll_topic/News_Train/Van_hoa.muc", 0),
("../data_conll_topic/News_Train/Kinh_te.muc", 1)]
dev_files = [("../data_conll_topic/News_Dev/Kinh_te.muc", 1)]
test_files = [("../data_conll_topic/News_Test/Kinh_te.muc", 2)]
ner.run(train_files, dev_files, test_files, 500)
def thethao_2_lstm():
print("thethao_2_lstm")
train_files = ["../folds/fold_1/Kinh_te.train"]
dev_files = ["../folds/fold_1/Kinh_te.dev"]
test_files = ["../folds/fold_1/Kinh_te.test"]
ner.run(train_files, dev_files, test_files, 500)
