def initialize(self): print('Data reader initialization ...') self.cursor = fever_db.get_cursor() # Prepare Data token_indexers = { 'tokens': \ SingleIdTokenIndexer(namespace='tokens'), 'elmo_chars': \ ELMoTokenCharactersIndexer(namespace='elmo_characters') } self.fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=cfg.lazy) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT \ / 'vocab_cache' \ / 'nli_basic') # THis is important ns = 'selection_labels' vocab.add_token_to_namespace('true', namespace=ns) vocab.add_token_to_namespace('false', namespace=ns) vocab.add_token_to_namespace('hidden', namespace=ns) vocab.change_token_with_index_to_namespace('hidden', -2, namespace=ns) # Label value vocab.get_index_to_token_vocabulary(ns) self.vocab = vocab self.weight_dict = weight_dict self.initialized = True
def spectrum_eval_manual_check(): batch_size = 64 lazy = True SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-17-12:10:35_mesim_elmo/i(34800)_epoch(5)_dev(0.5563056305630563)_loss(1.6648460462434564)_seed(12)" # IN_FILE = config.RESULT_PATH / "sent_retri_nn/2018_07_17_15:52:19_r/dev_sent.jsonl" IN_FILE = config.RESULT_PATH / "sent_retri_nn/2018_07_17_16:34:19_r/dev_sent.jsonl" # IN_FILE = config.RESULT_PATH / "sent_retri_nn/2018_07_17_16-34-19_r/dev_sent.jsonl" dev_sent_result_lsit = common.load_jsonl(IN_FILE) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) for sc_prob in [0.5, 0.7, 0.8, 0.9, 0.95, 0.98]: upstream_dev_list = score_converter_scaled(config.T_FEVER_DEV_JSONL, dev_sent_result_lsit, scale_prob=sc_prob, delete_prob=False) dev_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, upstream_dev_list) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) print("------------------------------------") print("Scaling_prob:", sc_prob) eval_mode = {'check_sent_id_correct': True, 'standard': True} print(c_scorer.fever_score(builded_dev_data, config.T_FEVER_DEV_JSONL, mode=eval_mode)) # del upstream_dev_list # del complete_upstream_dev_data del dev_fever_data_reader del dev_instances print("------------------------------------")
def eval_fever(): # save_path = "/home/easonnie/projects/MiscEnc/saved_models/06-07-21:58:06_esim_elmo/i(60900)_epoch(4)_um_dev(80.03458096013019)_m_dev(79.174732552216)_seed(12)" save_path = "/home/easonnie/projects/MiscEnc/saved_models/07-02-14:40:01_esim_elmo_linear_amr_cs_score_filtering_0.5/i(5900)_epoch(3)_um_dev(39.73759153783564)_m_dev(40.18339276617422)_seed(12)" # save_path = "/home/easonnie/projects/MiscEnc/saved_models/07-02-14:42:34_esim_elmo_cs_score_filtering_0.7/i(1300)_epoch(4)_um_dev(32.55695687550855)_m_dev(32.42995415180846)_seed(12)" batch_size = 32 # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } csnli_dataset_reader = CNLIReader(token_indexers=token_indexers, example_filter=lambda x: float(x['cs_score']) >= 0.7) # mnli_train_data_path = config.DATA_ROOT / "mnli/multinli_1.0_train.jsonl" mnli_m_dev_data_path = config.DATA_ROOT / "amrs/mnli_amr_ln/mnli_mdev.jsonl.cs" mnli_um_dev_data_path = config.DATA_ROOT / "amrs/mnli_amr_ln/mnli_umdev.jsonl.cs" # mnli_train_instances = csnli_dataset_reader.read(mnli_train_data_path) mnli_m_dev_instances = csnli_dataset_reader.read(mnli_m_dev_data_path) mnli_um_dev_instances = csnli_dataset_reader.read(mnli_um_dev_data_path) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300) model.load_state_dict(torch.load(save_path)) model.display() model.to(device) # Create Log File criterion = nn.CrossEntropyLoss() eval_iter = biterator(mnli_m_dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) m_dev_score, m_dev_loss = eval_model(model, eval_iter, criterion) eval_iter = biterator(mnli_um_dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) um_dev_score, um_dev_loss = eval_model(model, eval_iter, criterion) print(f"Dev(M):{m_dev_score}/{m_dev_loss}") print(f"Dev(UM):{um_dev_score}/{um_dev_loss}")
def hidden_eval_fever(): batch_size = 64 lazy = True SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-18-21:07:28_m_esim_wn_elmo_sample_fixed/i(57000)_epoch(8)_dev(0.5755075507550755)_loss(1.7175163737963839)_seed(12)" dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } p_dict = wn_persistent_api.persistence_load() dev_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) # dev_biterator = BasicIterator(batch_size=batch_size * 2) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size, 1024 + 300), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300) print("Model Max length:", model.max_l) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} for item in builded_dev_data: del item['label'] print(c_scorer.fever_score(builded_dev_data, common.load_jsonl(config.T_FEVER_DEV_JSONL), mode=eval_mode))
def pipeline_first_sent_selection(org_t_file, upstream_in_file, model_save_path): batch_size = 128 lazy = True SAVE_PATH = model_save_path print("Model From:", SAVE_PATH) dev_upstream_file = upstream_in_file # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy) complete_upstream_dev_data = get_full_list(org_t_file, dev_upstream_file, pred=True) print("Dev size:", len(complete_upstream_dev_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300, num_of_class=2) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) dev_sent_full_list = hidden_eval(model, eval_iter, complete_upstream_dev_data) return dev_sent_full_list
def get_score_multihop(t_data_file, additional_file, model_path, item_key='prioritized_docids_aside', top_k=6): batch_size = 64 lazy = True SAVE_PATH = model_path print("Model From:", SAVE_PATH) additional_sentence_list = get_additional_list(t_data_file, additional_file, item_key=item_key, top_k=top_k) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy) print("Additional Dev size:", len(additional_sentence_list)) dev_instances = dev_fever_data_reader.read(additional_sentence_list) # Load Vocabulary dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300, num_of_class=2) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) additional_sentence_list = hidden_eval(model, eval_iter, additional_sentence_list) return additional_sentence_list
def __init__(self, model_path): # Prepare Data lazy = False token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } p_dict = wn_persistent_api.persistence_load() dev_fever_data_reader = WNSIMIReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=420) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') # Build Model # device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) # device_num = -1 if device.type == 'cpu' else 0 device = torch.device("cpu") device_num = -1 if device.type == 'cpu' else 0 biterator = BasicIterator(batch_size=16) biterator.index_with(vocab) model = Model( rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size, 1024 + 450 + dev_fever_data_reader.wn_feature_size), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=400) model.display() model.to(device) model.load_state_dict(torch.load(model_path)) self.model = model self.dev_fever_data_reader = dev_fever_data_reader self.device_num = device_num self.biterator = biterator
def utest_data_loader(): num_epoch = 8 seed = 12 batch_size = 32 experiment_name = "mesim_wn_elmo" lazy = True dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } p_dict = wn_persistent_api.persistence_load() train_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict) dev_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model complete_upstream_train_data = get_sampled_data(config.T_FEVER_TRAIN_JSONL, train_upstream_file)[:20000] device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 sampled_train_instances = train_fever_data_reader.read(complete_upstream_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): pass batch['p_wn_feature'] batch['h_wn_feature'] # print(batch.keys()) # print(batch['p_wn_feature']) # print(batch['h_wn_feature']) wn_persistent_api.persistence_update(p_dict)
def hidden_eval_fever(): batch_size = 64 lazy = True SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-08-19:04:33_mesim_elmo/i(39700)_epoch(6)_dev(0.5251525152515252)_loss(1.5931938096682707)_seed(12)" dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } dev_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} print(c_scorer.fever_score(builded_dev_data, config.T_FEVER_DEV_JSONL, mode=eval_mode))
def hidden_eval_fever_adv_v1(): batch_size = 64 lazy = True dev_prob_threshold = 0.5 SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-20-22:28:24_mesim_wn_450_adv_sample_v1_|t_prob:0.35|top_k:8/i(46000)_epoch(7)_dev(0.6405140514051405)_loss(1.0761665150348825)_seed(12)" dev_upstream_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/2018_07_20_15:17:59_r/dev_sent.jsonl") # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } p_dict = wn_persistent_api.persistence_load() upstream_dev_list = score_converter_scaled(config.T_FEVER_DEV_JSONL, dev_upstream_sent_list, scale_prob=dev_prob_threshold, delete_prob=False) dev_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, upstream_dev_list) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model( rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size, 1024 + 450), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=300) print("Model Max length:", model.max_l) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} common.save_jsonl( builded_dev_data, config.RESULT_PATH / "nli_results" / "pipeline_results_1.jsonl") c_scorer.delete_label(builded_dev_data) print( c_scorer.fever_score(builded_dev_data, common.load_jsonl(config.FEVER_DEV_JSONL), mode=eval_mode))
def eval_for_remaining(): batch_size = 128 lazy = True # SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-16-11:37:07_simple_nn/i(25000)_epoch(1)_(tra_score:0.8188318831883188|clf_acc:95.67680650034835|pr:0.7394326932693269|rec:0.7282478247824783|f1:0.7337976403219241|loss:0.11368581993118955)" SAVE_PATH = config.PRO_ROOT / "saved_models/saved_sselector/i(57167)_epoch(6)_(tra_score:0.8850885088508851|raw_acc:1.0|pr:0.3834395939593578|rec:0.8276327632763276|f1:0.5240763176570098)_epoch" # SAVE_PATH = config.PRO_ROOT / "saved_models/07-20-01:35:16_simple_nn_startkp_0.4_de_0.05/i(53810)_epoch(4)_(tra_score:0.8577357735773578|raw_acc:1.0|pr:0.671477147714762|rec:0.7866036603660366|f1:0.7244953493898653)_epoch" print("Model From:", SAVE_PATH) # dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" # dev_upstream_file = config.RESULT_PATH / "doc_retri/cn_util_Jul17_docretri.singularize/dev.jsonl" # dev_upstream_file = config.RESULT_PATH / "doc_retri/docretri.pageview/dev.jsonl" # dev_upstream_file = config.RESULT_PATH / "doc_retri/docretri.pageview/train.jsonl" # # SAVE_RESULT_TARGET_FOLDER.mkdir() incoming_data_file = config.RESULT_PATH / "sent_retri_nn/remaining_training_cache/dev_s.jsonl" incoming_data = common.load_jsonl(incoming_data_file) SAVE_RESULT_TARGET_FOLDER = config.RESULT_PATH / "sent_retri_nn/remaining_training_cache" # out_file_name = "dev_sent.jsonl" out_file_name = "remain_dev_sent.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy) # complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True) # complete_upstream_dev_data = get_full_list(config.T_FEVER_TRAIN_JSONL, dev_upstream_file, pred=True) print("Dev size:", len(incoming_data)) dev_instances = dev_fever_data_reader.read(incoming_data) # Load Vocabulary dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300, num_of_class=2) model.load_state_dict(torch.load(SAVE_PATH)) model.display() model.to(device) eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) complete_upstream_dev_data = hidden_eval(model, eval_iter, incoming_data) common.save_jsonl(complete_upstream_dev_data, SAVE_RESULT_TARGET_FOLDER / out_file_name) total = 0 hit = 0 for item in complete_upstream_dev_data: assert item['selection_label'] == 'true' if item['prob'] >= 0.5: hit += 1 total += 1 print(hit, total, hit / total)
def train_fever_v2(): # train_fever_v1 is the old training script. # train_fever_v2 is the new training script created on 02 Oct 2018 11:40:24. # Here we keep the negative and positive portion to be consistent. num_epoch = 10 seed = 12 batch_size = 128 lazy = True torch.manual_seed(seed) keep_neg_sample_prob = 1 top_k_doc = 5 experiment_name = f"simple_nn_remain_{keep_neg_sample_prob}" # sample_prob_decay = 0.05 dev_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl" train_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/train_doc.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180) # dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False) dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180) complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True, top_k=top_k_doc) print("Dev size:", len(complete_upstream_dev_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=160, num_of_class=2) model.display() model.to(device) cloned_empty_model = copy.deepcopy(model) ema: EMA = EMA(parameters=model.named_parameters()) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() dev_actual_list = common.load_jsonl(config.T_FEVER_DEV_JSONL) for i_epoch in range(num_epoch): print("Resampling...") # Resampling complete_upstream_train_data = get_full_list( config.T_FEVER_TRAIN_JSONL, train_upstream_file, pred=False, top_k=top_k_doc) print("Sample Prob.:", keep_neg_sample_prob) filtered_train_data = post_filter(complete_upstream_train_data, keep_prob=keep_neg_sample_prob, seed=12 + i_epoch) # Change the seed to avoid duplicate sample... # keep_neg_sample_prob -= sample_prob_decay # if keep_neg_sample_prob <= 0: # keep_neg_sample_prob = 0.005 print("Sampled_length:", len(filtered_train_data)) sampled_train_instances = train_fever_data_reader.read( filtered_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) y = batch['selection_label'] loss = criterion(out, y) # No decay optimizer.zero_grad() loss.backward() optimizer.step() # Update EMA ema(model.named_parameters()) iteration += 1 if i_epoch <= 5: mod = 8000 else: mod = 8000 if iteration % mod == 0: eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) load_ema_to_model(cloned_empty_model, ema) # complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) # Only eval EMA complete_upstream_dev_data = hidden_eval( cloned_empty_model, eval_iter, complete_upstream_dev_data) dev_results_list = score_converter_v1( config.T_FEVER_DEV_JSONL, complete_upstream_dev_data, sent_retri_top_k=5, sent_retri_scal_prob=0.5) # This is only a wrapper for the simi_sampler eval_mode = {'check_sent_id_correct': True, 'standard': True} for a, b in zip(dev_actual_list, dev_results_list): b['predicted_label'] = a['label'] strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( dev_results_list, dev_actual_list, mode=eval_mode, verbose=False) tracking_score = strict_score print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}") print("Strict score:", strict_score) print(f"Eval Tracking score:", f"{tracking_score}") # need_save = False # if tracking_score > best_dev: # best_dev = tracking_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_' f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})_ema' ) save_ema_to_file(ema, save_path) # torch.save(model.state_dict(), save_path) print("Epoch Evaluation...") eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) load_ema_to_model(cloned_empty_model, ema) # complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) complete_upstream_dev_data = hidden_eval(cloned_empty_model, eval_iter, complete_upstream_dev_data) dev_results_list = score_converter_v1(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data, sent_retri_top_k=5, sent_retri_scal_prob=0.5) eval_mode = {'check_sent_id_correct': True, 'standard': True} for a, b in zip(dev_actual_list, dev_results_list): b['predicted_label'] = a['label'] strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( dev_results_list, dev_actual_list, mode=eval_mode, verbose=False) tracking_score = strict_score print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}") print("Strict score:", strict_score) print(f"Eval Tracking score:", f"{tracking_score}") if tracking_score > best_dev: best_dev = tracking_score save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_' f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})_epoch_ema' ) save_ema_to_file(ema, save_path)
def debug_fever(): num_epoch = 8 seed = 12 batch_size = 128 experiment_name = "simple_nn" lazy = True torch.manual_seed(seed) keep_neg_sample_prob = 0.6 sample_prob_decay = 0.1 dev_upstream_file = config.RESULT_PATH / "doc_retri/cn_util_Jul17_docretri.singularize/dev.jsonl" train_upstream_file = config.RESULT_PATH / "doc_retri/cn_util_Jul17_docretri.singularize/train.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=300) # dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False) dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=300) complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True) print("Dev size:", len(complete_upstream_dev_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=280, num_of_class=2) model.display() model.to(device) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 i_epoch = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) dev_results_list = score_converter_v0(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(dev_results_list, config.T_FEVER_DEV_JSONL, mode=eval_mode, verbose=False) total = len(dev_results_list) hit = eval_mode['check_sent_id_correct_hits'] tracking_score = hit / total print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/") print("Strict score:", strict_score) print(f"Eval Tracking score:", f"{tracking_score}") need_save = False if tracking_score > best_dev: best_dev = tracking_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_' f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})' ) torch.save(model.state_dict(), save_path) print("Epoch Evaluation...") eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) dev_results_list = score_converter_v0(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(dev_results_list, config.T_FEVER_DEV_JSONL, mode=eval_mode, verbose=False) total = len(dev_results_list) hit = eval_mode['check_sent_id_correct_hits'] tracking_score = hit / total print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/") print("Strict score:", strict_score) print(f"Eval Tracking score:", f"{tracking_score}") if tracking_score > best_dev: best_dev = tracking_score save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_' f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})_epoch' ) torch.save(model.state_dict(), save_path)
def train_fever_v1(): num_epoch = 10 seed = 12 batch_size = 128 dev_batch_size = 128 # experiment_name = "simple_nn_doc_first_sent" experiment_name = "simple_nn_doc" lazy = True torch.manual_seed(seed) contain_first_sentence = False pn_ratio = 1.0 # keep_neg_sample_prob = 0.4 # sample_prob_decay = 0.05 dev_upstream_file = config.RESULT_PATH / "doc_retri_bls/docretri.basic.nopageview/dev.jsonl" train_upstream_file = config.RESULT_PATH / "doc_retri_bls/docretri.basic.nopageview/train.jsonl" dev_data_list = common.load_jsonl(dev_upstream_file) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180) # dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False) dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180) cursor = fever_db.get_cursor() complete_upstream_dev_data = disamb.sample_disamb_inference(common.load_jsonl(dev_upstream_file), cursor, contain_first_sentence=contain_first_sentence) print("Dev size:", len(complete_upstream_dev_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=dev_batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=160, num_of_class=2) model.display() model.to(device) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() for i_epoch in range(num_epoch): print("Resampling...") # Resampling complete_upstream_train_data = disamb.sample_disamb_training_v0(common.load_jsonl(train_upstream_file), cursor, pn_ratio, contain_first_sentence) print("Sample Prob.:", pn_ratio) print("Sampled_length:", len(complete_upstream_train_data)) sampled_train_instances = train_fever_data_reader.read(complete_upstream_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) y = batch['selection_label'] loss = criterion(out, y) # No decay optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 if i_epoch <= 5: mod = 1000 else: mod = 500 if iteration % mod == 0: eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) disamb.enforce_disabuigation_into_retrieval_result_v0(complete_upstream_dev_data, dev_data_list) oracle_score, pr, rec, f1 = c_scorer.fever_doc_only(dev_data_list, dev_data_list, max_evidence=5) print(f"Dev(raw_acc/pr/rec/f1):{oracle_score}/{pr}/{rec}/{f1}") print("Strict score:", oracle_score) print(f"Eval Tracking score:", f"{oracle_score}") need_save = False if oracle_score > best_dev: best_dev = oracle_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_' f'(tra_score:{oracle_score}|pr:{pr}|rec:{rec}|f1:{f1})' ) torch.save(model.state_dict(), save_path) # print("Epoch Evaluation...") eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) disamb.enforce_disabuigation_into_retrieval_result_v0(complete_upstream_dev_data, dev_data_list) oracle_score, pr, rec, f1 = c_scorer.fever_doc_only(dev_data_list, dev_data_list, max_evidence=5) print(f"Dev(raw_acc/pr/rec/f1):{oracle_score}/{pr}/{rec}/{f1}") print("Strict score:", oracle_score) print(f"Eval Tracking score:", f"{oracle_score}") need_save = False if oracle_score > best_dev: best_dev = oracle_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_e' f'(tra_score:{oracle_score}|pr:{pr}|rec:{rec}|f1:{f1})' ) torch.save(model.state_dict(), save_path)
def train_fever(): num_epoch = 8 seed = 12 batch_size = 128 experiment_name = "simple_nn" lazy = True torch.manual_seed(seed) keep_neg_sample_prob = 0.5 sample_prob_decay = 0.1 dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy) # dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False) dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy) complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True) print("Dev size:", len(complete_upstream_dev_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300, num_of_class=2) model.display() model.to(device) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() for i_epoch in range(num_epoch): print("Resampling...") # Resampling complete_upstream_train_data = get_full_list(config.T_FEVER_TRAIN_JSONL, train_upstream_file, pred=False) filtered_train_data = post_filter(complete_upstream_train_data, keep_prob=keep_neg_sample_prob, seed=12 + i_epoch) # Change the seed to avoid duplicate sample... keep_neg_sample_prob -= sample_prob_decay print("Sampled_length:", len(filtered_train_data)) sampled_train_instances = train_fever_data_reader.read(filtered_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) y = batch['selection_label'] loss = criterion(out, y) # No decay optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 if i_epoch <= 4: mod = 25000 else: mod = 10000 if iteration % mod == 0: eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) dev_score, dev_loss, complete_upstream_dev_data = full_eval_model(model, eval_iter, criterion, complete_upstream_dev_data) dev_results_list = score_converter_v0(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(dev_results_list, config.T_FEVER_DEV_JSONL, mode=eval_mode, verbose=False) total = len(dev_results_list) hit = eval_mode['check_sent_id_correct_hits'] tracking_score = hit / total print(f"Dev(clf_acc/pr/rec/f1/loss):{dev_score}/{pr}/{rec}/{f1}/{dev_loss}") print(f"Tracking score:", f"{tracking_score}") need_save = False if tracking_score > best_dev: best_dev = tracking_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_' f'(tra_score:{tracking_score}|clf_acc:{dev_score}|pr:{pr}|rec:{rec}|f1:{f1}|loss:{dev_loss})' ) torch.save(model.state_dict(), save_path)
def debug_fever(): num_epoch = 8 seed = 12 batch_size = 32 experiment_name = "simple_nn" lazy = True torch.manual_seed(seed) dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy) # dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False) dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False) complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True) print("Dev size:", len(complete_upstream_dev_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=batch_size * 4) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=600, num_of_class=2) model.display() model.to(device) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() eval_iter = dev_biterator(dev_instances[:256], shuffle=False, num_epochs=1, cuda_device=device_num) dev_score, dev_loss, complete_upstream_dev_data = full_eval_model( model, eval_iter, criterion, complete_upstream_dev_data[:256]) print(complete_upstream_dev_data[:256])
def train_fever(): num_epoch = 8 seed = 12 batch_size = 32 experiment_name = "mesim_elmo" lazy = True dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl" train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy, max_l=360) dev_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy, max_l=360) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=300) model.display() model.to(device) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") best_dev = -1 iteration = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() for i_epoch in range(num_epoch): print("Resampling...") # Resampling complete_upstream_train_data = get_sampled_data(config.T_FEVER_TRAIN_JSONL, train_upstream_file) sampled_train_instances = train_fever_data_reader.read(complete_upstream_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) y = batch['label'] loss = criterion(out, y) # No decay optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 if i_epoch <= 4: mod = 5000 else: mod = 200 if iteration % mod == 0: eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) dev_score, dev_loss = full_eval_model(model, eval_iter, criterion, complete_upstream_dev_data) print(f"Dev:{dev_score}/{dev_loss}") need_save = False if dev_score > best_dev: best_dev = dev_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_dev({dev_score})_loss({dev_loss})_seed({seed})' ) torch.save(model.state_dict(), save_path)
def train_fever_std_ema_v1(resume_model=None, do_analysis=False): """ This method is created on 26 Nov 2018 08:50 with the purpose of training vc and ss all together. :param resume_model: :param wn_feature: :return: """ num_epoch = 200 seed = 12 batch_size = 32 lazy = True train_prob_threshold = 0.02 train_sample_top_k = 8 dev_prob_threshold = 0.1 dev_sample_top_k = 5 top_k_doc = 5 schedule_sample_dict = defaultdict(lambda: 0.1) ratio_ss_for_vc = 0.2 schedule_sample_dict.update({ 0: 0.1, 1: 0.1, # 200k + 400K 2: 0.1, 3: 0.1, # 200k + 200k ~ 200k + 100k 4: 0.1, 5: 0.1, # 200k + 100k 6: 0.1 # 20k + 20k }) # Eval at beginning of the training. eval_full_epoch = 1 eval_nei_epoches = [2, 3, 4, 5, 6, 7] neg_only = False debug = False experiment_name = f"vc_ss_v17_ratio_ss_for_vc:{ratio_ss_for_vc}|t_prob:{train_prob_threshold}|top_k:{train_sample_top_k}_scheduled_neg_sampler" # resume_model = None print("Do EMA:") print("Dev prob threshold:", dev_prob_threshold) print("Train prob threshold:", train_prob_threshold) print("Train sample top k:", train_sample_top_k) # Get upstream sentence document retrieval data dev_doc_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl" train_doc_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/train_doc.jsonl" complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_doc_upstream_file, pred=True, top_k=top_k_doc) complete_upstream_train_data = get_full_list(config.T_FEVER_TRAIN_JSONL, train_doc_upstream_file, pred=False, top_k=top_k_doc) if debug: complete_upstream_dev_data = complete_upstream_dev_data[:1000] complete_upstream_train_data = complete_upstream_train_data[:1000] print("Dev size:", len(complete_upstream_dev_data)) print("Train size:", len(complete_upstream_train_data)) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } # Data Reader dev_fever_data_reader = VCSS_Reader(token_indexers=token_indexers, lazy=lazy, max_l=260) train_fever_data_reader = VCSS_Reader(token_indexers=token_indexers, lazy=lazy, max_l=260) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.add_token_to_namespace('true', namespace='labels') vocab.add_token_to_namespace('false', namespace='labels') vocab.add_token_to_namespace("hidden", namespace="labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Reader and prepare end vc_ss_training_sampler = VCSSTrainingSampler(complete_upstream_train_data) vc_ss_training_sampler.show_info() # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(rnn_size_in=(1024 + 300 + 1, 1024 + 450 + 1), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=300, num_of_class=4) print("Model Max length:", model.max_l) if resume_model is not None: model.load_state_dict(torch.load(resume_model)) model.display() model.to(device) cloned_empty_model = copy.deepcopy(model) ema: EMA = EMA(parameters=model.named_parameters()) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() analysis_dir = None if do_analysis: analysis_dir = Path(file_path_prefix) / "analysis_aux" analysis_dir.mkdir() # Save source code end. # Staring parameter setup best_dev = -1 iteration = 0 start_lr = 0.0001 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() # parameter setup end for i_epoch in range(num_epoch): print("Resampling...") # This is for train # This is for sample candidate data for from result of ss for vc. # This we will need to do after each epoch. if i_epoch == eval_full_epoch: # only eval at 1 print("We now need to eval the whole training set.") print("Be patient and hope good luck!") load_ema_to_model(cloned_empty_model, ema) eval_sent_for_sampler(cloned_empty_model, token_indexers, vocab, vc_ss_training_sampler) elif i_epoch in eval_nei_epoches: # at 2, 3, 4 eval for NEI print("We now need to eval the NEI training set.") print("Be patient and hope good luck!") load_ema_to_model(cloned_empty_model, ema) eval_sent_for_sampler(cloned_empty_model, token_indexers, vocab, vc_ss_training_sampler, nei_only=True) train_data_with_candidate_sample_list = vc_ss.data_wrangler.sample_sentences_for_vc_with_nei( config.T_FEVER_TRAIN_JSONL, vc_ss_training_sampler.sent_list, train_prob_threshold, train_sample_top_k) # We initialize the prob for each sentence so the sampler can work, but we will need to run the model for dev data to work. train_selection_dict = paired_selection_score_dict( vc_ss_training_sampler.sent_list) cur_train_vc_data = adv_simi_sample_with_prob_v1_1( config.T_FEVER_TRAIN_JSONL, train_data_with_candidate_sample_list, train_selection_dict, tokenized=True) if do_analysis: # Customized analysis output common.save_jsonl( vc_ss_training_sampler.sent_list, analysis_dir / f"E_{i_epoch}_whole_train_sent_{save_tool.get_cur_time_str()}.jsonl" ) common.save_jsonl( train_data_with_candidate_sample_list, analysis_dir / f"E_{i_epoch}_sampled_train_sent_{save_tool.get_cur_time_str()}.jsonl" ) common.save_jsonl( cur_train_vc_data, analysis_dir / f"E_{i_epoch}_train_vc_data_{save_tool.get_cur_time_str()}.jsonl" ) print(f"E{i_epoch} VC_data:", len(cur_train_vc_data)) # This is for sample negative candidate data for ss # After sampling, we decrease the ratio. neg_sample_upper_prob = schedule_sample_dict[i_epoch] print("Neg Sampler upper rate:", neg_sample_upper_prob) # print("Rate decreasing") # neg_sample_upper_prob -= decay_r neg_sample_upper_prob = max(0.000, neg_sample_upper_prob) cur_train_ss_data = vc_ss_training_sampler.sample_for_ss( neg_only=neg_only, upper_prob=neg_sample_upper_prob) if i_epoch >= 1: # if epoch num >= 6 we balance pos and neg example for selection # new_ss_data = [] pos_ss_data = [] neg_ss_data = [] for item in cur_train_ss_data: if item['selection_label'] == 'true': pos_ss_data.append(item) elif item['selection_label'] == 'false': neg_ss_data.append(item) ss_sample_size = min(len(pos_ss_data), len(neg_ss_data)) random.shuffle(pos_ss_data) random.shuffle(neg_ss_data) cur_train_ss_data = pos_ss_data[:int( ss_sample_size * 0.5)] + neg_ss_data[:ss_sample_size] random.shuffle(cur_train_ss_data) vc_ss_training_sampler.show_info(cur_train_ss_data) print(f"E{i_epoch} SS_data:", len(cur_train_ss_data)) vc_ss.data_wrangler.assign_task_label(cur_train_ss_data, 'ss') vc_ss.data_wrangler.assign_task_label(cur_train_vc_data, 'vc') vs_ss_train_list = cur_train_ss_data + cur_train_vc_data random.shuffle(vs_ss_train_list) print(f"E{i_epoch} Total ss+vc:", len(vs_ss_train_list)) vc_ss_instance = train_fever_data_reader.read(vs_ss_train_list) train_iter = biterator(vc_ss_instance, shuffle=True, num_epochs=1) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) if i_epoch >= 1: ratio_ss_for_vc = 0.8 loss = compute_mixing_loss( model, out, batch, criterion, vc_ss_training_sampler, ss_for_vc_prob=ratio_ss_for_vc) # Important change # No decay optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 # EMA update ema(model.named_parameters()) if i_epoch < 9: mod = 10000 # mod = 100 else: mod = 2000 if iteration % mod == 0: # This is the code for eval: load_ema_to_model(cloned_empty_model, ema) vc_ss.data_wrangler.assign_task_label( complete_upstream_dev_data, 'ss') dev_ss_instance = dev_fever_data_reader.read( complete_upstream_dev_data) eval_ss_iter = biterator(dev_ss_instance, num_epochs=1, shuffle=False) scored_dev_sent_data = hidden_eval_ss( cloned_empty_model, eval_ss_iter, complete_upstream_dev_data) # for vc filtered_dev_list = vc_ss.data_wrangler.sample_sentences_for_vc_with_nei( config.T_FEVER_DEV_JSONL, scored_dev_sent_data, dev_prob_threshold, dev_sample_top_k) dev_selection_dict = paired_selection_score_dict( scored_dev_sent_data) ready_dev_list = select_sent_with_prob_for_eval( config.T_FEVER_DEV_JSONL, filtered_dev_list, dev_selection_dict, tokenized=True) vc_ss.data_wrangler.assign_task_label(ready_dev_list, 'vc') dev_vc_instance = dev_fever_data_reader.read(ready_dev_list) eval_vc_iter = biterator(dev_vc_instance, num_epochs=1, shuffle=False) eval_dev_result_list = hidden_eval_vc(cloned_empty_model, eval_vc_iter, ready_dev_list) # Scoring eval_mode = {'check_sent_id_correct': True, 'standard': True} strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( eval_dev_result_list, common.load_jsonl(config.T_FEVER_DEV_JSONL), mode=eval_mode, verbose=False) print("Fever Score(Strict/Acc./Precision/Recall/F1):", strict_score, acc_score, pr, rec, f1) print(f"Dev:{strict_score}/{acc_score}") if do_analysis: # Customized analysis output common.save_jsonl( scored_dev_sent_data, analysis_dir / f"E_{i_epoch}_scored_dev_sent_{save_tool.get_cur_time_str()}.jsonl" ) common.save_jsonl( eval_dev_result_list, analysis_dir / f"E_{i_epoch}_eval_vc_output_data_{save_tool.get_cur_time_str()}.jsonl" ) need_save = False if strict_score > best_dev: best_dev = strict_score need_save = True if need_save or i_epoch < 7: # save_path = os.path.join( # file_path_prefix, # f'i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_seed({seed})' # ) # torch.save(model.state_dict(), save_path) ema_save_path = os.path.join( file_path_prefix, f'ema_i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_p({pr})_r({rec})_f1({f1})_seed({seed})' ) save_ema_to_file(ema, ema_save_path)
def analysis_model(model_path): batch_size = 32 lazy = True train_prob_threshold = 0.02 train_sample_top_k = 8 dev_prob_threshold = 0.1 dev_sample_top_k = 5 neg_sample_upper_prob = 0.006 decay_r = 0.002 top_k_doc = 5 dev_doc_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl" complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_doc_upstream_file, pred=True, top_k=top_k_doc) print("Dev size:", len(complete_upstream_dev_data)) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } # Data Reader dev_fever_data_reader = VCSS_Reader(token_indexers=token_indexers, lazy=lazy, max_l=260) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.add_token_to_namespace('true', namespace='labels') vocab.add_token_to_namespace('false', namespace='labels') vocab.add_token_to_namespace("hidden", namespace="labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Reader and prepare end # vc_ss_training_sampler = VCSSTrainingSampler(complete_upstream_train_data) # vc_ss_training_sampler.show_info() # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(rnn_size_in=(1024 + 300 + 1, 1024 + 450 + 1), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=300) print("Model Max length:", model.max_l) model.display() model.to(device) cloned_empty_model = copy.deepcopy(model) load_ema_to_model(cloned_empty_model, model_path) vc_ss.data_wrangler.assign_task_label(complete_upstream_dev_data, 'ss') dev_ss_instance = dev_fever_data_reader.read(complete_upstream_dev_data) eval_ss_iter = biterator(dev_ss_instance, num_epochs=1, shuffle=False) scored_dev_sent_data = hidden_eval_ss(cloned_empty_model, eval_ss_iter, complete_upstream_dev_data) common.save_jsonl(scored_dev_sent_data, "dev_scored_sent_data.jsonl") # for vc filtered_dev_list = vc_ss.data_wrangler.sample_sentences_for_vc_with_nei( config.T_FEVER_DEV_JSONL, scored_dev_sent_data, dev_prob_threshold, dev_sample_top_k) common.save_jsonl(filtered_dev_list, "dev_scored_sent_data_after_sample.jsonl") dev_selection_dict = paired_selection_score_dict(scored_dev_sent_data) ready_dev_list = select_sent_with_prob_for_eval(config.T_FEVER_DEV_JSONL, filtered_dev_list, dev_selection_dict, tokenized=True) vc_ss.data_wrangler.assign_task_label(ready_dev_list, 'vc') dev_vc_instance = dev_fever_data_reader.read(ready_dev_list) eval_vc_iter = biterator(dev_vc_instance, num_epochs=1, shuffle=False) eval_dev_result_list = hidden_eval_vc(cloned_empty_model, eval_vc_iter, ready_dev_list) common.save_jsonl(eval_dev_result_list, "dev_nli_results.jsonl") # Scoring eval_mode = {'check_sent_id_correct': True, 'standard': True} strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( eval_dev_result_list, common.load_jsonl(config.T_FEVER_DEV_JSONL), mode=eval_mode, verbose=False) print("Fever Score(Strict/Acc./Precision/Recall/F1):", strict_score, acc_score, pr, rec, f1) print(f"Dev:{strict_score}/{acc_score}")
def train_fever_v1_advsample(): num_epoch = 12 seed = 12 batch_size = 32 lazy = True dev_prob_threshold = 0.5 train_prob_threshold = 0.35 train_sample_top_k = 10 experiment_name = f"mesim_wn_450_adv_sample_v1_|t_prob:{train_prob_threshold}|top_k:{train_sample_top_k}" print("Dev prob threshold:", dev_prob_threshold) print("Train prob threshold:", train_prob_threshold) print("Train sample top k:", train_sample_top_k) dev_upstream_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/2018_07_20_15:17:59_r/dev_sent.jsonl") train_upstream_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/2018_07_20_15:17:59_r/train_sent.jsonl") # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } p_dict = wn_persistent_api.persistence_load() upstream_dev_list = score_converter_scaled(config.T_FEVER_DEV_JSONL, dev_upstream_sent_list, scale_prob=dev_prob_threshold, delete_prob=False) dev_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360) train_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360) complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, upstream_dev_list) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=batch_size * 2) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model( rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size, 1024 + 450), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=300) print("Model Max length:", model.max_l) model.display() model.to(device) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 start_lr = 0.0002 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() for i_epoch in range(num_epoch): print("Resampling...") # Resampling # complete_upstream_train_data = get_sampled_data(config.T_FEVER_TRAIN_JSONL, train_upstream_file) complete_upstream_train_data = get_adv_sampled_data( config.T_FEVER_TRAIN_JSONL, train_upstream_sent_list, threshold_prob=train_prob_threshold, top_n=train_sample_top_k) print("Sample data length:", len(complete_upstream_train_data)) sampled_train_instances = train_fever_data_reader.read( complete_upstream_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) y = batch['label'] loss = criterion(out, y) # No decay optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 if i_epoch <= 6: # mod = 5000 mod = 5000 else: mod = 500 if iteration % mod == 0: eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) dev_score, dev_loss = full_eval_model( model, eval_iter, criterion, complete_upstream_dev_data) print(f"Dev:{dev_score}/{dev_loss}") need_save = False if dev_score > best_dev: best_dev = dev_score need_save = True if need_save: save_path = os.path.join( file_path_prefix, f'i({iteration})_epoch({i_epoch})_dev({dev_score})_loss({dev_loss})_seed({seed})' ) torch.save(model.state_dict(), save_path) # Save some cache wordnet feature. wn_persistent_api.persistence_update(p_dict)
def train_fever_std_ema_v1(resume_model=None, wn_feature=False): """ This method is the new training script for train fever with span and probability score. :param resume_model: :param wn_feature: :return: """ num_epoch = 200 seed = 12 batch_size = 32 lazy = True dev_prob_threshold = 0.1 train_prob_threshold = 0.1 train_sample_top_k = 8 experiment_name = f"nsmn_sent_wise_std_ema_lr1|t_prob:{train_prob_threshold}|top_k:{train_sample_top_k}" # resume_model = None print("Do EMA:") print("Dev prob threshold:", dev_prob_threshold) print("Train prob threshold:", train_prob_threshold) print("Train sample top k:", train_sample_top_k) dev_upstream_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/balanced_sentence_selection_results/dev_sent_pred_scores.jsonl" ) train_upstream_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/balanced_sentence_selection_results/train_sent_scores.jsonl" ) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } print("Building Prob Dicts...") train_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/balanced_sentence_selection_results/train_sent_scores.jsonl" ) dev_sent_list = common.load_jsonl( config.RESULT_PATH / "sent_retri_nn/balanced_sentence_selection_results/dev_sent_pred_scores.jsonl" ) selection_dict = paired_selection_score_dict(train_sent_list) selection_dict = paired_selection_score_dict(dev_sent_list, selection_dict) upstream_dev_list = threshold_sampler_insure_unique( config.T_FEVER_DEV_JSONL, dev_upstream_sent_list, prob_threshold=dev_prob_threshold, top_n=5) # Specifiy ablation to remove wordnet and number embeddings. dev_fever_data_reader = WNSIMIReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=320, ablation=None) train_fever_data_reader = WNSIMIReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=320, shuffle_sentences=False, ablation=None) complete_upstream_dev_data = select_sent_with_prob_for_eval( config.T_FEVER_DEV_JSONL, upstream_dev_list, selection_dict, tokenized=True) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model( rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size, 1024 + 450 + dev_fever_data_reader.wn_feature_size), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=300, use_extra_lex_feature=False, max_span_l=100) print("Model Max length:", model.max_l) if resume_model is not None: model.load_state_dict(torch.load(resume_model)) model.display() model.to(device) cloned_empty_model = copy.deepcopy(model) ema: EMA = EMA(parameters=model.named_parameters()) # Create Log File file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}") # Save the source code. script_name = os.path.basename(__file__) with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it: out_f.write(it.read()) out_f.flush() # Save source code end. best_dev = -1 iteration = 0 start_lr = 0.0001 optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr) criterion = nn.CrossEntropyLoss() for i_epoch in range(num_epoch): print("Resampling...") # Resampling train_data_with_candidate_sample_list = \ threshold_sampler_insure_unique(config.T_FEVER_TRAIN_JSONL, train_upstream_sent_list, train_prob_threshold, top_n=train_sample_top_k) complete_upstream_train_data = adv_simi_sample_with_prob_v1_1( config.T_FEVER_TRAIN_JSONL, train_data_with_candidate_sample_list, selection_dict, tokenized=True) print("Sample data length:", len(complete_upstream_train_data)) sampled_train_instances = train_fever_data_reader.read( complete_upstream_train_data) train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num) for i, batch in tqdm(enumerate(train_iter)): model.train() out = model(batch) y = batch['label'] loss = criterion(out, y) # No decay optimizer.zero_grad() loss.backward() optimizer.step() iteration += 1 # EMA update ema(model.named_parameters()) if i_epoch < 15: mod = 10000 # mod = 10 else: mod = 2000 if iteration % mod == 0: # eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) # complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) # # eval_mode = {'check_sent_id_correct': True, 'standard': True} # strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(complete_upstream_dev_data, # common.load_jsonl(config.T_FEVER_DEV_JSONL), # mode=eval_mode, # verbose=False) # print("Fever Score(Strict/Acc./Precision/Recall/F1):", strict_score, acc_score, pr, rec, f1) # # print(f"Dev:{strict_score}/{acc_score}") # EMA saving eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) load_ema_to_model(cloned_empty_model, ema) complete_upstream_dev_data = hidden_eval( cloned_empty_model, eval_iter, complete_upstream_dev_data) eval_mode = {'check_sent_id_correct': True, 'standard': True} strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( complete_upstream_dev_data, common.load_jsonl(config.T_FEVER_DEV_JSONL), mode=eval_mode, verbose=False) print("Fever Score EMA(Strict/Acc./Precision/Recall/F1):", strict_score, acc_score, pr, rec, f1) print(f"Dev EMA:{strict_score}/{acc_score}") need_save = False if strict_score > best_dev: best_dev = strict_score need_save = True if need_save: # save_path = os.path.join( # file_path_prefix, # f'i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_seed({seed})' # ) # torch.save(model.state_dict(), save_path) ema_save_path = os.path.join( file_path_prefix, f'ema_i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_seed({seed})' ) save_ema_to_file(ema, ema_save_path)
def eval_and_save_v2(model_path, is_ema, saving_dir, save_train_data=True, prob_thresholds=0.5): # This method was modified on 21 NOV 2018 # for evaluating balanced trained selection model with different threshold value. # It will then be used for later verification. # Evaluate and Save all the sentence pairs results to be used for downstream verificaion # 03 Oct 2018 03:56:40. seed = 12 batch_size = 128 lazy = True torch.manual_seed(seed) keep_neg_sample_prob = 1 top_k_doc = 5 # sample_prob_decay = 0.05 dev_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl" train_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/train_doc.jsonl" # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer( namespace='elmo_characters') # This is the elmo_characters } train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180) dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180) complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True, top_k=top_k_doc) complete_upstream_train_data = get_full_list(config.T_FEVER_TRAIN_JSONL, train_upstream_file, pred=False, top_k=top_k_doc) print("Dev size:", len(complete_upstream_dev_data)) print("Train size:", len(complete_upstream_train_data)) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) train_instances = train_fever_data_reader.read( complete_upstream_train_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) dev_biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") # THis is important vocab.add_token_to_namespace("true", namespace="selection_labels") vocab.add_token_to_namespace("false", namespace="selection_labels") vocab.add_token_to_namespace("hidden", namespace="selection_labels") vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels') # Label value vocab.get_index_to_token_vocabulary('selection_labels') print(vocab.get_token_to_index_vocabulary('selection_labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) dev_biterator.index_with(vocab) # exit(0) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), embedding_dim=300, max_l=160, num_of_class=2) if not is_ema: model.load_state_dict(torch.load(model_path)) else: load_ema_to_model(model, model_path) model.display() model.to(device) dev_actual_list = common.load_jsonl(config.T_FEVER_DEV_JSONL) train_actual_list = common.load_jsonl(config.T_FEVER_TRAIN_JSONL) eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1) train_iter = biterator(train_instances, shuffle=False, num_epochs=1) complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) if save_train_data: complete_upstream_train_data = hidden_eval( model, train_iter, complete_upstream_train_data) common.save_jsonl(complete_upstream_train_data, Path(str(saving_dir)) / "train_sent_scores.jsonl") common.save_jsonl(complete_upstream_dev_data, Path(str(saving_dir)) / "dev_sent_pred_scores.jsonl") if not isinstance(prob_thresholds, list): prob_thresholds = [prob_thresholds] for scal_prob in prob_thresholds: print("Eval Dev Data prob_threshold:", scal_prob) dev_results_list = score_converter_v1(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data, sent_retri_top_k=5, sent_retri_scal_prob=scal_prob) # This is only a wrapper for the simi_sampler eval_mode = {'check_sent_id_correct': True, 'standard': True} for a, b in zip(dev_actual_list, dev_results_list): b['predicted_label'] = a['label'] strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( dev_results_list, dev_actual_list, mode=eval_mode, verbose=False) tracking_score = strict_score print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/") print("Strict score:", strict_score) print(f"Eval Tracking score:", f"{tracking_score}") if save_train_data: print("Build Train Data") train_results_list = score_converter_v1( config.T_FEVER_TRAIN_JSONL, complete_upstream_train_data, sent_retri_top_k=5, sent_retri_scal_prob=prob_threshold) # This is only a wrapper for the simi_sampler eval_mode = {'check_sent_id_correct': True, 'standard': True} for a, b in zip(train_actual_list, train_results_list): b['predicted_label'] = a['label'] strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score( train_results_list, train_actual_list, mode=eval_mode, verbose=False) tracking_score = strict_score print(f"Train(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/") print("Strict score:", strict_score) print(f"Eval Tracking score:", f"{tracking_score}")
build_vocab_embeddings(vocab, config.DATA_ROOT / "embeddings/glove.840B.300d.txt", embd_dim=300, saved_path=config.DATA_ROOT / "vocab_cache" / "nli_basic") if __name__ == '__main__': # nli_path_list = [] # for in_file in (config.DATA_ROOT / "mnli").iterdir(): # nli_path_list.append(in_file) # # for in_file in (config.DATA_ROOT / "snli").iterdir(): # nli_path_list.append(in_file) # # build_fever_vocab_with_embeddings_and_save() # build_vocab_embeddings(vocab, config.DATA_ROOT / "embeddings/glove.840B.300d.txt", # embd_dim=300, saved_path=config.DATA_ROOT / "vocab_cache" / "nli") vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") print(weight_dict) print(vocab) # print(vocab.get_vocab_size()) # vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') # # print(vocab.get_token_to_index_vocabulary('labels')) # print(vocab.get_index_to_token_vocabulary('labels')) # print() # print(weight_dict['glove.840B.300d'].size()) #
def pipeline_nli_run(t_org_file, upstream_dev_data_list, upstream_sent_file_list, model_path): batch_size = 32 lazy = True print("Size:", len(upstream_dev_data_list)) print("Building Prob Dicts...") selection_dict = dict() for upstream_sent_file in upstream_sent_file_list: upstream_sent_l = common.load_jsonl(upstream_sent_file) selection_dict = paired_selection_score_dict(upstream_sent_l, selection_dict) # Prepare Data token_indexers = { 'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens 'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters } p_dict = wn_persistent_api.persistence_load() dev_fever_data_reader = WNSIMIReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360) complete_upstream_dev_data = select_sent_with_prob_for_eval(t_org_file, upstream_dev_data_list, selection_dict, tokenized=True, pipeline=True) complete_upstream_dev_data = append_hidden_label(complete_upstream_dev_data) dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data) # Load Vocabulary biterator = BasicIterator(batch_size=batch_size) vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic") vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels') print(vocab.get_token_to_index_vocabulary('labels')) print(vocab.get_vocab_size('tokens')) biterator.index_with(vocab) # Build Model device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0) device_num = -1 if device.type == 'cpu' else 0 model = Model(rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size, 1024 + 450), rnn_size_out=(450, 450), weight=weight_dict['glove.840B.300d'], vocab_size=vocab.get_vocab_size('tokens'), mlp_d=900, embedding_dim=300, max_l=300) print("Model Max length:", model.max_l) model.load_state_dict(torch.load(model_path)) model.display() model.to(device) eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num) complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data) wn_persistent_api.persistence_update(p_dict) return complete_upstream_dev_data