def initialize(self):
print('Data reader initialization ...')
self.cursor = fever_db.get_cursor()
# Prepare Data
token_indexers = {
'tokens': \
SingleIdTokenIndexer(namespace='tokens'),
'elmo_chars': \
ELMoTokenCharactersIndexer(namespace='elmo_characters')
}
self.fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=cfg.lazy)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT \
/ 'vocab_cache' \
/ 'nli_basic')
# THis is important
ns = 'selection_labels'
vocab.add_token_to_namespace('true', namespace=ns)
vocab.add_token_to_namespace('false', namespace=ns)
vocab.add_token_to_namespace('hidden', namespace=ns)
vocab.change_token_with_index_to_namespace('hidden', -2, namespace=ns)
# Label value
vocab.get_index_to_token_vocabulary(ns)
self.vocab = vocab
self.weight_dict = weight_dict
self.initialized = True
def spectrum_eval_manual_check():
batch_size = 64
lazy = True
SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-17-12:10:35_mesim_elmo/i(34800)_epoch(5)_dev(0.5563056305630563)_loss(1.6648460462434564)_seed(12)"
# IN_FILE = config.RESULT_PATH / "sent_retri_nn/2018_07_17_15:52:19_r/dev_sent.jsonl"
IN_FILE = config.RESULT_PATH / "sent_retri_nn/2018_07_17_16:34:19_r/dev_sent.jsonl"
# IN_FILE = config.RESULT_PATH / "sent_retri_nn/2018_07_17_16-34-19_r/dev_sent.jsonl"
dev_sent_result_lsit = common.load_jsonl(IN_FILE)
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
for sc_prob in [0.5, 0.7, 0.8, 0.9, 0.95, 0.98]:
upstream_dev_list = score_converter_scaled(config.T_FEVER_DEV_JSONL, dev_sent_result_lsit, scale_prob=sc_prob,
delete_prob=False)
dev_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, upstream_dev_list)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
print("------------------------------------")
print("Scaling_prob:", sc_prob)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
print(c_scorer.fever_score(builded_dev_data, config.T_FEVER_DEV_JSONL, mode=eval_mode))
# del upstream_dev_list
# del complete_upstream_dev_data
del dev_fever_data_reader
del dev_instances
print("------------------------------------")
def eval_fever():
# save_path = "/home/easonnie/projects/MiscEnc/saved_models/06-07-21:58:06_esim_elmo/i(60900)_epoch(4)_um_dev(80.03458096013019)_m_dev(79.174732552216)_seed(12)"
save_path = "/home/easonnie/projects/MiscEnc/saved_models/07-02-14:40:01_esim_elmo_linear_amr_cs_score_filtering_0.5/i(5900)_epoch(3)_um_dev(39.73759153783564)_m_dev(40.18339276617422)_seed(12)"
# save_path = "/home/easonnie/projects/MiscEnc/saved_models/07-02-14:42:34_esim_elmo_cs_score_filtering_0.7/i(1300)_epoch(4)_um_dev(32.55695687550855)_m_dev(32.42995415180846)_seed(12)"
batch_size = 32
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
csnli_dataset_reader = CNLIReader(token_indexers=token_indexers,
example_filter=lambda x: float(x['cs_score']) >= 0.7)
# mnli_train_data_path = config.DATA_ROOT / "mnli/multinli_1.0_train.jsonl"
mnli_m_dev_data_path = config.DATA_ROOT / "amrs/mnli_amr_ln/mnli_mdev.jsonl.cs"
mnli_um_dev_data_path = config.DATA_ROOT / "amrs/mnli_amr_ln/mnli_umdev.jsonl.cs"
# mnli_train_instances = csnli_dataset_reader.read(mnli_train_data_path)
mnli_m_dev_instances = csnli_dataset_reader.read(mnli_m_dev_data_path)
mnli_um_dev_instances = csnli_dataset_reader.read(mnli_um_dev_data_path)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300)
model.load_state_dict(torch.load(save_path))
model.display()
model.to(device)
# Create Log File
criterion = nn.CrossEntropyLoss()
eval_iter = biterator(mnli_m_dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
m_dev_score, m_dev_loss = eval_model(model, eval_iter, criterion)
eval_iter = biterator(mnli_um_dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
um_dev_score, um_dev_loss = eval_model(model, eval_iter, criterion)
print(f"Dev(M):{m_dev_score}/{m_dev_loss}")
print(f"Dev(UM):{um_dev_score}/{um_dev_loss}")
def hidden_eval_fever():
batch_size = 64
lazy = True
SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-18-21:07:28_m_esim_wn_elmo_sample_fixed/i(57000)_epoch(8)_dev(0.5755075507550755)_loss(1.7175163737963839)_seed(12)"
dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
p_dict = wn_persistent_api.persistence_load()
dev_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
# dev_biterator = BasicIterator(batch_size=batch_size * 2)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size,
1024 + 300),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300)
print("Model Max length:", model.max_l)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
for item in builded_dev_data:
del item['label']
print(c_scorer.fever_score(builded_dev_data, common.load_jsonl(config.T_FEVER_DEV_JSONL), mode=eval_mode))
def pipeline_first_sent_selection(org_t_file, upstream_in_file, model_save_path):
batch_size = 128
lazy = True
SAVE_PATH = model_save_path
print("Model From:", SAVE_PATH)
dev_upstream_file = upstream_in_file
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy)
complete_upstream_dev_data = get_full_list(org_t_file, dev_upstream_file, pred=True)
print("Dev size:", len(complete_upstream_dev_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300, num_of_class=2)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
dev_sent_full_list = hidden_eval(model, eval_iter, complete_upstream_dev_data)
return dev_sent_full_list
def get_score_multihop(t_data_file, additional_file, model_path, item_key='prioritized_docids_aside', top_k=6):
batch_size = 64
lazy = True
SAVE_PATH = model_path
print("Model From:", SAVE_PATH)
additional_sentence_list = get_additional_list(t_data_file, additional_file, item_key=item_key, top_k=top_k)
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy)
print("Additional Dev size:", len(additional_sentence_list))
dev_instances = dev_fever_data_reader.read(additional_sentence_list)
# Load Vocabulary
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300, num_of_class=2)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
additional_sentence_list = hidden_eval(model, eval_iter, additional_sentence_list)
return additional_sentence_list
def __init__(self, model_path):
# Prepare Data
lazy = False
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
p_dict = wn_persistent_api.persistence_load()
dev_fever_data_reader = WNSIMIReader(token_indexers=token_indexers,
lazy=lazy,
wn_p_dict=p_dict,
max_l=420)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden',
-2,
namespace='labels')
# Build Model
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
# device_num = -1 if device.type == 'cpu' else 0
device = torch.device("cpu")
device_num = -1 if device.type == 'cpu' else 0
biterator = BasicIterator(batch_size=16)
biterator.index_with(vocab)
model = Model(
rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size,
1024 + 450 + dev_fever_data_reader.wn_feature_size),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300,
max_l=400)
model.display()
model.to(device)
model.load_state_dict(torch.load(model_path))
self.model = model
self.dev_fever_data_reader = dev_fever_data_reader
self.device_num = device_num
self.biterator = biterator
def utest_data_loader():
num_epoch = 8
seed = 12
batch_size = 32
experiment_name = "mesim_wn_elmo"
lazy = True
dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl"
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
p_dict = wn_persistent_api.persistence_load()
train_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict)
dev_fever_data_reader = WNReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
complete_upstream_train_data = get_sampled_data(config.T_FEVER_TRAIN_JSONL, train_upstream_file)[:20000]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
sampled_train_instances = train_fever_data_reader.read(complete_upstream_train_data)
train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
pass
batch['p_wn_feature']
batch['h_wn_feature']
# print(batch.keys())
# print(batch['p_wn_feature'])
# print(batch['h_wn_feature'])
wn_persistent_api.persistence_update(p_dict)
def hidden_eval_fever():
batch_size = 64
lazy = True
SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-08-19:04:33_mesim_elmo/i(39700)_epoch(6)_dev(0.5251525152515252)_loss(1.5931938096682707)_seed(12)"
dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
dev_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
builded_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
print(c_scorer.fever_score(builded_dev_data, config.T_FEVER_DEV_JSONL, mode=eval_mode))
def hidden_eval_fever_adv_v1():
batch_size = 64
lazy = True
dev_prob_threshold = 0.5
SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-20-22:28:24_mesim_wn_450_adv_sample_v1_|t_prob:0.35|top_k:8/i(46000)_epoch(7)_dev(0.6405140514051405)_loss(1.0761665150348825)_seed(12)"
dev_upstream_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/2018_07_20_15:17:59_r/dev_sent.jsonl")
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
p_dict = wn_persistent_api.persistence_load()
upstream_dev_list = score_converter_scaled(config.T_FEVER_DEV_JSONL,
dev_upstream_sent_list,
scale_prob=dev_prob_threshold,
delete_prob=False)
dev_fever_data_reader = WNReader(token_indexers=token_indexers,
lazy=lazy,
wn_p_dict=p_dict,
max_l=360)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL,
upstream_dev_list)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden',
-2,
namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(
rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size,
1024 + 450),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300,
max_l=300)
print("Model Max length:", model.max_l)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
eval_iter = biterator(dev_instances,
shuffle=False,
num_epochs=1,
cuda_device=device_num)
builded_dev_data = hidden_eval(model, eval_iter,
complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
common.save_jsonl(
builded_dev_data,
config.RESULT_PATH / "nli_results" / "pipeline_results_1.jsonl")
c_scorer.delete_label(builded_dev_data)
print(
c_scorer.fever_score(builded_dev_data,
common.load_jsonl(config.FEVER_DEV_JSONL),
mode=eval_mode))
def eval_for_remaining():
batch_size = 128
lazy = True
# SAVE_PATH = "/home/easonnie/projects/FunEver/saved_models/07-16-11:37:07_simple_nn/i(25000)_epoch(1)_(tra_score:0.8188318831883188|clf_acc:95.67680650034835|pr:0.7394326932693269|rec:0.7282478247824783|f1:0.7337976403219241|loss:0.11368581993118955)"
SAVE_PATH = config.PRO_ROOT / "saved_models/saved_sselector/i(57167)_epoch(6)_(tra_score:0.8850885088508851|raw_acc:1.0|pr:0.3834395939593578|rec:0.8276327632763276|f1:0.5240763176570098)_epoch"
# SAVE_PATH = config.PRO_ROOT / "saved_models/07-20-01:35:16_simple_nn_startkp_0.4_de_0.05/i(53810)_epoch(4)_(tra_score:0.8577357735773578|raw_acc:1.0|pr:0.671477147714762|rec:0.7866036603660366|f1:0.7244953493898653)_epoch"
print("Model From:", SAVE_PATH)
# dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
# dev_upstream_file = config.RESULT_PATH / "doc_retri/cn_util_Jul17_docretri.singularize/dev.jsonl"
# dev_upstream_file = config.RESULT_PATH / "doc_retri/docretri.pageview/dev.jsonl"
# dev_upstream_file = config.RESULT_PATH / "doc_retri/docretri.pageview/train.jsonl"
#
# SAVE_RESULT_TARGET_FOLDER.mkdir()
incoming_data_file = config.RESULT_PATH / "sent_retri_nn/remaining_training_cache/dev_s.jsonl"
incoming_data = common.load_jsonl(incoming_data_file)
SAVE_RESULT_TARGET_FOLDER = config.RESULT_PATH / "sent_retri_nn/remaining_training_cache"
# out_file_name = "dev_sent.jsonl"
out_file_name = "remain_dev_sent.jsonl"
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=lazy)
# complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True)
# complete_upstream_dev_data = get_full_list(config.T_FEVER_TRAIN_JSONL, dev_upstream_file, pred=True)
print("Dev size:", len(incoming_data))
dev_instances = dev_fever_data_reader.read(incoming_data)
# Load Vocabulary
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden",
-2,
namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300,
max_l=300,
num_of_class=2)
model.load_state_dict(torch.load(SAVE_PATH))
model.display()
model.to(device)
eval_iter = dev_biterator(dev_instances,
shuffle=False,
num_epochs=1,
cuda_device=device_num)
complete_upstream_dev_data = hidden_eval(model, eval_iter, incoming_data)
common.save_jsonl(complete_upstream_dev_data,
SAVE_RESULT_TARGET_FOLDER / out_file_name)
total = 0
hit = 0
for item in complete_upstream_dev_data:
assert item['selection_label'] == 'true'
if item['prob'] >= 0.5:
hit += 1
total += 1
print(hit, total, hit / total)
def train_fever_v2():
# train_fever_v1 is the old training script.
# train_fever_v2 is the new training script created on 02 Oct 2018 11:40:24.
# Here we keep the negative and positive portion to be consistent.
num_epoch = 10
seed = 12
batch_size = 128
lazy = True
torch.manual_seed(seed)
keep_neg_sample_prob = 1
top_k_doc = 5
experiment_name = f"simple_nn_remain_{keep_neg_sample_prob}"
# sample_prob_decay = 0.05
dev_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl"
train_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/train_doc.jsonl"
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=lazy,
max_l=180)
# dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False)
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=lazy,
max_l=180)
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL,
dev_upstream_file,
pred=True,
top_k=top_k_doc)
print("Dev size:", len(complete_upstream_dev_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden",
-2,
namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300,
max_l=160,
num_of_class=2)
model.display()
model.to(device)
cloned_empty_model = copy.deepcopy(model)
ema: EMA = EMA(parameters=model.named_parameters())
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name),
'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=start_lr)
criterion = nn.CrossEntropyLoss()
dev_actual_list = common.load_jsonl(config.T_FEVER_DEV_JSONL)
for i_epoch in range(num_epoch):
print("Resampling...")
# Resampling
complete_upstream_train_data = get_full_list(
config.T_FEVER_TRAIN_JSONL,
train_upstream_file,
pred=False,
top_k=top_k_doc)
print("Sample Prob.:", keep_neg_sample_prob)
filtered_train_data = post_filter(complete_upstream_train_data,
keep_prob=keep_neg_sample_prob,
seed=12 + i_epoch)
# Change the seed to avoid duplicate sample...
# keep_neg_sample_prob -= sample_prob_decay
# if keep_neg_sample_prob <= 0:
# keep_neg_sample_prob = 0.005
print("Sampled_length:", len(filtered_train_data))
sampled_train_instances = train_fever_data_reader.read(
filtered_train_data)
train_iter = biterator(sampled_train_instances,
shuffle=True,
num_epochs=1,
cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
y = batch['selection_label']
loss = criterion(out, y)
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Update EMA
ema(model.named_parameters())
iteration += 1
if i_epoch <= 5:
mod = 8000
else:
mod = 8000
if iteration % mod == 0:
eval_iter = dev_biterator(dev_instances,
shuffle=False,
num_epochs=1,
cuda_device=device_num)
load_ema_to_model(cloned_empty_model, ema)
# complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
# Only eval EMA
complete_upstream_dev_data = hidden_eval(
cloned_empty_model, eval_iter, complete_upstream_dev_data)
dev_results_list = score_converter_v1(
config.T_FEVER_DEV_JSONL,
complete_upstream_dev_data,
sent_retri_top_k=5,
sent_retri_scal_prob=0.5)
# This is only a wrapper for the simi_sampler
eval_mode = {'check_sent_id_correct': True, 'standard': True}
for a, b in zip(dev_actual_list, dev_results_list):
b['predicted_label'] = a['label']
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
dev_results_list,
dev_actual_list,
mode=eval_mode,
verbose=False)
tracking_score = strict_score
print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}")
print("Strict score:", strict_score)
print(f"Eval Tracking score:", f"{tracking_score}")
# need_save = False
# if tracking_score > best_dev:
# best_dev = tracking_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix, f'i({iteration})_epoch({i_epoch})_'
f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})_ema'
)
save_ema_to_file(ema, save_path)
# torch.save(model.state_dict(), save_path)
print("Epoch Evaluation...")
eval_iter = dev_biterator(dev_instances,
shuffle=False,
num_epochs=1,
cuda_device=device_num)
load_ema_to_model(cloned_empty_model, ema)
# complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
complete_upstream_dev_data = hidden_eval(cloned_empty_model, eval_iter,
complete_upstream_dev_data)
dev_results_list = score_converter_v1(config.T_FEVER_DEV_JSONL,
complete_upstream_dev_data,
sent_retri_top_k=5,
sent_retri_scal_prob=0.5)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
for a, b in zip(dev_actual_list, dev_results_list):
b['predicted_label'] = a['label']
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
dev_results_list, dev_actual_list, mode=eval_mode, verbose=False)
tracking_score = strict_score
print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}")
print("Strict score:", strict_score)
print(f"Eval Tracking score:", f"{tracking_score}")
if tracking_score > best_dev:
best_dev = tracking_score
save_path = os.path.join(
file_path_prefix, f'i({iteration})_epoch({i_epoch})_'
f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})_epoch_ema'
)
save_ema_to_file(ema, save_path)
def debug_fever():
num_epoch = 8
seed = 12
batch_size = 128
experiment_name = "simple_nn"
lazy = True
torch.manual_seed(seed)
keep_neg_sample_prob = 0.6
sample_prob_decay = 0.1
dev_upstream_file = config.RESULT_PATH / "doc_retri/cn_util_Jul17_docretri.singularize/dev.jsonl"
train_upstream_file = config.RESULT_PATH / "doc_retri/cn_util_Jul17_docretri.singularize/train.jsonl"
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=300)
# dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False)
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=300)
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True)
print("Dev size:", len(complete_upstream_dev_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=280, num_of_class=2)
model.display()
model.to(device)
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
i_epoch = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr)
criterion = nn.CrossEntropyLoss()
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
dev_results_list = score_converter_v0(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(dev_results_list, config.T_FEVER_DEV_JSONL,
mode=eval_mode, verbose=False)
total = len(dev_results_list)
hit = eval_mode['check_sent_id_correct_hits']
tracking_score = hit / total
print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/")
print("Strict score:", strict_score)
print(f"Eval Tracking score:", f"{tracking_score}")
need_save = False
if tracking_score > best_dev:
best_dev = tracking_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_'
f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})'
)
torch.save(model.state_dict(), save_path)
print("Epoch Evaluation...")
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
dev_results_list = score_converter_v0(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(dev_results_list, config.T_FEVER_DEV_JSONL,
mode=eval_mode, verbose=False)
total = len(dev_results_list)
hit = eval_mode['check_sent_id_correct_hits']
tracking_score = hit / total
print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/")
print("Strict score:", strict_score)
print(f"Eval Tracking score:", f"{tracking_score}")
if tracking_score > best_dev:
best_dev = tracking_score
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_'
f'(tra_score:{tracking_score}|raw_acc:{acc_score}|pr:{pr}|rec:{rec}|f1:{f1})_epoch'
)
torch.save(model.state_dict(), save_path)
def train_fever_v1():
num_epoch = 10
seed = 12
batch_size = 128
dev_batch_size = 128
# experiment_name = "simple_nn_doc_first_sent"
experiment_name = "simple_nn_doc"
lazy = True
torch.manual_seed(seed)
contain_first_sentence = False
pn_ratio = 1.0
# keep_neg_sample_prob = 0.4
# sample_prob_decay = 0.05
dev_upstream_file = config.RESULT_PATH / "doc_retri_bls/docretri.basic.nopageview/dev.jsonl"
train_upstream_file = config.RESULT_PATH / "doc_retri_bls/docretri.basic.nopageview/train.jsonl"
dev_data_list = common.load_jsonl(dev_upstream_file)
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180)
# dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False)
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy, max_l=180)
cursor = fever_db.get_cursor()
complete_upstream_dev_data = disamb.sample_disamb_inference(common.load_jsonl(dev_upstream_file), cursor,
contain_first_sentence=contain_first_sentence)
print("Dev size:", len(complete_upstream_dev_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=dev_batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=160, num_of_class=2)
model.display()
model.to(device)
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr)
criterion = nn.CrossEntropyLoss()
for i_epoch in range(num_epoch):
print("Resampling...")
# Resampling
complete_upstream_train_data = disamb.sample_disamb_training_v0(common.load_jsonl(train_upstream_file),
cursor, pn_ratio, contain_first_sentence)
print("Sample Prob.:", pn_ratio)
print("Sampled_length:", len(complete_upstream_train_data))
sampled_train_instances = train_fever_data_reader.read(complete_upstream_train_data)
train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
y = batch['selection_label']
loss = criterion(out, y)
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
iteration += 1
if i_epoch <= 5:
mod = 1000
else:
mod = 500
if iteration % mod == 0:
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
disamb.enforce_disabuigation_into_retrieval_result_v0(complete_upstream_dev_data,
dev_data_list)
oracle_score, pr, rec, f1 = c_scorer.fever_doc_only(dev_data_list, dev_data_list, max_evidence=5)
print(f"Dev(raw_acc/pr/rec/f1):{oracle_score}/{pr}/{rec}/{f1}")
print("Strict score:", oracle_score)
print(f"Eval Tracking score:", f"{oracle_score}")
need_save = False
if oracle_score > best_dev:
best_dev = oracle_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_'
f'(tra_score:{oracle_score}|pr:{pr}|rec:{rec}|f1:{f1})'
)
torch.save(model.state_dict(), save_path)
#
print("Epoch Evaluation...")
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
disamb.enforce_disabuigation_into_retrieval_result_v0(complete_upstream_dev_data,
dev_data_list)
oracle_score, pr, rec, f1 = c_scorer.fever_doc_only(dev_data_list, dev_data_list, max_evidence=5)
print(f"Dev(raw_acc/pr/rec/f1):{oracle_score}/{pr}/{rec}/{f1}")
print("Strict score:", oracle_score)
print(f"Eval Tracking score:", f"{oracle_score}")
need_save = False
if oracle_score > best_dev:
best_dev = oracle_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_e'
f'(tra_score:{oracle_score}|pr:{pr}|rec:{rec}|f1:{f1})'
)
torch.save(model.state_dict(), save_path)
def train_fever():
num_epoch = 8
seed = 12
batch_size = 128
experiment_name = "simple_nn"
lazy = True
torch.manual_seed(seed)
keep_neg_sample_prob = 0.5
sample_prob_decay = 0.1
dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl"
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy)
# dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False)
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=lazy)
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL, dev_upstream_file, pred=True)
print("Dev size:", len(complete_upstream_dev_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden", -2, namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300, num_of_class=2)
model.display()
model.to(device)
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name), 'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr)
criterion = nn.CrossEntropyLoss()
for i_epoch in range(num_epoch):
print("Resampling...")
# Resampling
complete_upstream_train_data = get_full_list(config.T_FEVER_TRAIN_JSONL, train_upstream_file, pred=False)
filtered_train_data = post_filter(complete_upstream_train_data, keep_prob=keep_neg_sample_prob,
seed=12 + i_epoch)
# Change the seed to avoid duplicate sample...
keep_neg_sample_prob -= sample_prob_decay
print("Sampled_length:", len(filtered_train_data))
sampled_train_instances = train_fever_data_reader.read(filtered_train_data)
train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
y = batch['selection_label']
loss = criterion(out, y)
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
iteration += 1
if i_epoch <= 4:
mod = 25000
else:
mod = 10000
if iteration % mod == 0:
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
dev_score, dev_loss, complete_upstream_dev_data = full_eval_model(model, eval_iter, criterion,
complete_upstream_dev_data)
dev_results_list = score_converter_v0(config.T_FEVER_DEV_JSONL, complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(dev_results_list, config.T_FEVER_DEV_JSONL,
mode=eval_mode, verbose=False)
total = len(dev_results_list)
hit = eval_mode['check_sent_id_correct_hits']
tracking_score = hit / total
print(f"Dev(clf_acc/pr/rec/f1/loss):{dev_score}/{pr}/{rec}/{f1}/{dev_loss}")
print(f"Tracking score:", f"{tracking_score}")
need_save = False
if tracking_score > best_dev:
best_dev = tracking_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_'
f'(tra_score:{tracking_score}|clf_acc:{dev_score}|pr:{pr}|rec:{rec}|f1:{f1}|loss:{dev_loss})'
)
torch.save(model.state_dict(), save_path)
def debug_fever():
num_epoch = 8
seed = 12
batch_size = 32
experiment_name = "simple_nn"
lazy = True
torch.manual_seed(seed)
dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl"
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=lazy)
# dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers, lazy=False)
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=False)
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL,
dev_upstream_file,
pred=True)
print("Dev size:", len(complete_upstream_dev_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=batch_size * 4)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden",
-2,
namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300,
max_l=600,
num_of_class=2)
model.display()
model.to(device)
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name),
'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=start_lr)
criterion = nn.CrossEntropyLoss()
eval_iter = dev_biterator(dev_instances[:256],
shuffle=False,
num_epochs=1,
cuda_device=device_num)
dev_score, dev_loss, complete_upstream_dev_data = full_eval_model(
model, eval_iter, criterion, complete_upstream_dev_data[:256])
print(complete_upstream_dev_data[:256])
def train_fever():
num_epoch = 8
seed = 12
batch_size = 32
experiment_name = "mesim_elmo"
lazy = True
dev_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/dev.jsonl"
train_upstream_file = config.RESULT_PATH / "sent_retri/2018_07_05_17:17:50_r/train.jsonl"
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy, max_l=360)
dev_fever_data_reader = BasicReader(token_indexers=token_indexers, lazy=lazy, max_l=360)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL, dev_upstream_file)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300, max_l=300)
model.display()
model.to(device)
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
best_dev = -1
iteration = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=start_lr)
criterion = nn.CrossEntropyLoss()
for i_epoch in range(num_epoch):
print("Resampling...")
# Resampling
complete_upstream_train_data = get_sampled_data(config.T_FEVER_TRAIN_JSONL, train_upstream_file)
sampled_train_instances = train_fever_data_reader.read(complete_upstream_train_data)
train_iter = biterator(sampled_train_instances, shuffle=True, num_epochs=1, cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
y = batch['label']
loss = criterion(out, y)
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
iteration += 1
if i_epoch <= 4:
mod = 5000
else:
mod = 200
if iteration % mod == 0:
eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
dev_score, dev_loss = full_eval_model(model, eval_iter, criterion, complete_upstream_dev_data)
print(f"Dev:{dev_score}/{dev_loss}")
need_save = False
if dev_score > best_dev:
best_dev = dev_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_dev({dev_score})_loss({dev_loss})_seed({seed})'
)
torch.save(model.state_dict(), save_path)
def train_fever_std_ema_v1(resume_model=None, do_analysis=False):
"""
This method is created on 26 Nov 2018 08:50 with the purpose of training vc and ss all together.
:param resume_model:
:param wn_feature:
:return:
"""
num_epoch = 200
seed = 12
batch_size = 32
lazy = True
train_prob_threshold = 0.02
train_sample_top_k = 8
dev_prob_threshold = 0.1
dev_sample_top_k = 5
top_k_doc = 5
schedule_sample_dict = defaultdict(lambda: 0.1)
ratio_ss_for_vc = 0.2
schedule_sample_dict.update({
0: 0.1,
1: 0.1, # 200k + 400K
2: 0.1,
3: 0.1, # 200k + 200k ~ 200k + 100k
4: 0.1,
5: 0.1, # 200k + 100k
6: 0.1 # 20k + 20k
})
# Eval at beginning of the training.
eval_full_epoch = 1
eval_nei_epoches = [2, 3, 4, 5, 6, 7]
neg_only = False
debug = False
experiment_name = f"vc_ss_v17_ratio_ss_for_vc:{ratio_ss_for_vc}|t_prob:{train_prob_threshold}|top_k:{train_sample_top_k}_scheduled_neg_sampler"
# resume_model = None
print("Do EMA:")
print("Dev prob threshold:", dev_prob_threshold)
print("Train prob threshold:", train_prob_threshold)
print("Train sample top k:", train_sample_top_k)
# Get upstream sentence document retrieval data
dev_doc_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl"
train_doc_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/train_doc.jsonl"
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL,
dev_doc_upstream_file,
pred=True,
top_k=top_k_doc)
complete_upstream_train_data = get_full_list(config.T_FEVER_TRAIN_JSONL,
train_doc_upstream_file,
pred=False,
top_k=top_k_doc)
if debug:
complete_upstream_dev_data = complete_upstream_dev_data[:1000]
complete_upstream_train_data = complete_upstream_train_data[:1000]
print("Dev size:", len(complete_upstream_dev_data))
print("Train size:", len(complete_upstream_train_data))
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
# Data Reader
dev_fever_data_reader = VCSS_Reader(token_indexers=token_indexers,
lazy=lazy,
max_l=260)
train_fever_data_reader = VCSS_Reader(token_indexers=token_indexers,
lazy=lazy,
max_l=260)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
vocab.add_token_to_namespace('true', namespace='labels')
vocab.add_token_to_namespace('false', namespace='labels')
vocab.add_token_to_namespace("hidden", namespace="labels")
vocab.change_token_with_index_to_namespace("hidden",
-2,
namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Reader and prepare end
vc_ss_training_sampler = VCSSTrainingSampler(complete_upstream_train_data)
vc_ss_training_sampler.show_info()
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(rnn_size_in=(1024 + 300 + 1, 1024 + 450 + 1),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300,
max_l=300,
num_of_class=4)
print("Model Max length:", model.max_l)
if resume_model is not None:
model.load_state_dict(torch.load(resume_model))
model.display()
model.to(device)
cloned_empty_model = copy.deepcopy(model)
ema: EMA = EMA(parameters=model.named_parameters())
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name),
'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
analysis_dir = None
if do_analysis:
analysis_dir = Path(file_path_prefix) / "analysis_aux"
analysis_dir.mkdir()
# Save source code end.
# Staring parameter setup
best_dev = -1
iteration = 0
start_lr = 0.0001
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=start_lr)
criterion = nn.CrossEntropyLoss()
# parameter setup end
for i_epoch in range(num_epoch):
print("Resampling...")
# This is for train
# This is for sample candidate data for from result of ss for vc.
# This we will need to do after each epoch.
if i_epoch == eval_full_epoch: # only eval at 1
print("We now need to eval the whole training set.")
print("Be patient and hope good luck!")
load_ema_to_model(cloned_empty_model, ema)
eval_sent_for_sampler(cloned_empty_model, token_indexers, vocab,
vc_ss_training_sampler)
elif i_epoch in eval_nei_epoches: # at 2, 3, 4 eval for NEI
print("We now need to eval the NEI training set.")
print("Be patient and hope good luck!")
load_ema_to_model(cloned_empty_model, ema)
eval_sent_for_sampler(cloned_empty_model,
token_indexers,
vocab,
vc_ss_training_sampler,
nei_only=True)
train_data_with_candidate_sample_list = vc_ss.data_wrangler.sample_sentences_for_vc_with_nei(
config.T_FEVER_TRAIN_JSONL, vc_ss_training_sampler.sent_list,
train_prob_threshold, train_sample_top_k)
# We initialize the prob for each sentence so the sampler can work, but we will need to run the model for dev data to work.
train_selection_dict = paired_selection_score_dict(
vc_ss_training_sampler.sent_list)
cur_train_vc_data = adv_simi_sample_with_prob_v1_1(
config.T_FEVER_TRAIN_JSONL,
train_data_with_candidate_sample_list,
train_selection_dict,
tokenized=True)
if do_analysis:
# Customized analysis output
common.save_jsonl(
vc_ss_training_sampler.sent_list, analysis_dir /
f"E_{i_epoch}_whole_train_sent_{save_tool.get_cur_time_str()}.jsonl"
)
common.save_jsonl(
train_data_with_candidate_sample_list, analysis_dir /
f"E_{i_epoch}_sampled_train_sent_{save_tool.get_cur_time_str()}.jsonl"
)
common.save_jsonl(
cur_train_vc_data, analysis_dir /
f"E_{i_epoch}_train_vc_data_{save_tool.get_cur_time_str()}.jsonl"
)
print(f"E{i_epoch} VC_data:", len(cur_train_vc_data))
# This is for sample negative candidate data for ss
# After sampling, we decrease the ratio.
neg_sample_upper_prob = schedule_sample_dict[i_epoch]
print("Neg Sampler upper rate:", neg_sample_upper_prob)
# print("Rate decreasing")
# neg_sample_upper_prob -= decay_r
neg_sample_upper_prob = max(0.000, neg_sample_upper_prob)
cur_train_ss_data = vc_ss_training_sampler.sample_for_ss(
neg_only=neg_only, upper_prob=neg_sample_upper_prob)
if i_epoch >= 1: # if epoch num >= 6 we balance pos and neg example for selection
# new_ss_data = []
pos_ss_data = []
neg_ss_data = []
for item in cur_train_ss_data:
if item['selection_label'] == 'true':
pos_ss_data.append(item)
elif item['selection_label'] == 'false':
neg_ss_data.append(item)
ss_sample_size = min(len(pos_ss_data), len(neg_ss_data))
random.shuffle(pos_ss_data)
random.shuffle(neg_ss_data)
cur_train_ss_data = pos_ss_data[:int(
ss_sample_size * 0.5)] + neg_ss_data[:ss_sample_size]
random.shuffle(cur_train_ss_data)
vc_ss_training_sampler.show_info(cur_train_ss_data)
print(f"E{i_epoch} SS_data:", len(cur_train_ss_data))
vc_ss.data_wrangler.assign_task_label(cur_train_ss_data, 'ss')
vc_ss.data_wrangler.assign_task_label(cur_train_vc_data, 'vc')
vs_ss_train_list = cur_train_ss_data + cur_train_vc_data
random.shuffle(vs_ss_train_list)
print(f"E{i_epoch} Total ss+vc:", len(vs_ss_train_list))
vc_ss_instance = train_fever_data_reader.read(vs_ss_train_list)
train_iter = biterator(vc_ss_instance, shuffle=True, num_epochs=1)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
if i_epoch >= 1:
ratio_ss_for_vc = 0.8
loss = compute_mixing_loss(
model,
out,
batch,
criterion,
vc_ss_training_sampler,
ss_for_vc_prob=ratio_ss_for_vc) # Important change
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
iteration += 1
# EMA update
ema(model.named_parameters())
if i_epoch < 9:
mod = 10000
# mod = 100
else:
mod = 2000
if iteration % mod == 0:
# This is the code for eval:
load_ema_to_model(cloned_empty_model, ema)
vc_ss.data_wrangler.assign_task_label(
complete_upstream_dev_data, 'ss')
dev_ss_instance = dev_fever_data_reader.read(
complete_upstream_dev_data)
eval_ss_iter = biterator(dev_ss_instance,
num_epochs=1,
shuffle=False)
scored_dev_sent_data = hidden_eval_ss(
cloned_empty_model, eval_ss_iter,
complete_upstream_dev_data)
# for vc
filtered_dev_list = vc_ss.data_wrangler.sample_sentences_for_vc_with_nei(
config.T_FEVER_DEV_JSONL, scored_dev_sent_data,
dev_prob_threshold, dev_sample_top_k)
dev_selection_dict = paired_selection_score_dict(
scored_dev_sent_data)
ready_dev_list = select_sent_with_prob_for_eval(
config.T_FEVER_DEV_JSONL,
filtered_dev_list,
dev_selection_dict,
tokenized=True)
vc_ss.data_wrangler.assign_task_label(ready_dev_list, 'vc')
dev_vc_instance = dev_fever_data_reader.read(ready_dev_list)
eval_vc_iter = biterator(dev_vc_instance,
num_epochs=1,
shuffle=False)
eval_dev_result_list = hidden_eval_vc(cloned_empty_model,
eval_vc_iter,
ready_dev_list)
# Scoring
eval_mode = {'check_sent_id_correct': True, 'standard': True}
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
eval_dev_result_list,
common.load_jsonl(config.T_FEVER_DEV_JSONL),
mode=eval_mode,
verbose=False)
print("Fever Score(Strict/Acc./Precision/Recall/F1):",
strict_score, acc_score, pr, rec, f1)
print(f"Dev:{strict_score}/{acc_score}")
if do_analysis:
# Customized analysis output
common.save_jsonl(
scored_dev_sent_data, analysis_dir /
f"E_{i_epoch}_scored_dev_sent_{save_tool.get_cur_time_str()}.jsonl"
)
common.save_jsonl(
eval_dev_result_list, analysis_dir /
f"E_{i_epoch}_eval_vc_output_data_{save_tool.get_cur_time_str()}.jsonl"
)
need_save = False
if strict_score > best_dev:
best_dev = strict_score
need_save = True
if need_save or i_epoch < 7:
# save_path = os.path.join(
# file_path_prefix,
# f'i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_seed({seed})'
# )
# torch.save(model.state_dict(), save_path)
ema_save_path = os.path.join(
file_path_prefix,
f'ema_i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_p({pr})_r({rec})_f1({f1})_seed({seed})'
)
save_ema_to_file(ema, ema_save_path)
def analysis_model(model_path):
batch_size = 32
lazy = True
train_prob_threshold = 0.02
train_sample_top_k = 8
dev_prob_threshold = 0.1
dev_sample_top_k = 5
neg_sample_upper_prob = 0.006
decay_r = 0.002
top_k_doc = 5
dev_doc_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl"
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL,
dev_doc_upstream_file,
pred=True,
top_k=top_k_doc)
print("Dev size:", len(complete_upstream_dev_data))
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
# Data Reader
dev_fever_data_reader = VCSS_Reader(token_indexers=token_indexers,
lazy=lazy,
max_l=260)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
vocab.add_token_to_namespace('true', namespace='labels')
vocab.add_token_to_namespace('false', namespace='labels')
vocab.add_token_to_namespace("hidden", namespace="labels")
vocab.change_token_with_index_to_namespace("hidden",
-2,
namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Reader and prepare end
# vc_ss_training_sampler = VCSSTrainingSampler(complete_upstream_train_data)
# vc_ss_training_sampler.show_info()
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(rnn_size_in=(1024 + 300 + 1, 1024 + 450 + 1),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300,
max_l=300)
print("Model Max length:", model.max_l)
model.display()
model.to(device)
cloned_empty_model = copy.deepcopy(model)
load_ema_to_model(cloned_empty_model, model_path)
vc_ss.data_wrangler.assign_task_label(complete_upstream_dev_data, 'ss')
dev_ss_instance = dev_fever_data_reader.read(complete_upstream_dev_data)
eval_ss_iter = biterator(dev_ss_instance, num_epochs=1, shuffle=False)
scored_dev_sent_data = hidden_eval_ss(cloned_empty_model, eval_ss_iter,
complete_upstream_dev_data)
common.save_jsonl(scored_dev_sent_data, "dev_scored_sent_data.jsonl")
# for vc
filtered_dev_list = vc_ss.data_wrangler.sample_sentences_for_vc_with_nei(
config.T_FEVER_DEV_JSONL, scored_dev_sent_data, dev_prob_threshold,
dev_sample_top_k)
common.save_jsonl(filtered_dev_list,
"dev_scored_sent_data_after_sample.jsonl")
dev_selection_dict = paired_selection_score_dict(scored_dev_sent_data)
ready_dev_list = select_sent_with_prob_for_eval(config.T_FEVER_DEV_JSONL,
filtered_dev_list,
dev_selection_dict,
tokenized=True)
vc_ss.data_wrangler.assign_task_label(ready_dev_list, 'vc')
dev_vc_instance = dev_fever_data_reader.read(ready_dev_list)
eval_vc_iter = biterator(dev_vc_instance, num_epochs=1, shuffle=False)
eval_dev_result_list = hidden_eval_vc(cloned_empty_model, eval_vc_iter,
ready_dev_list)
common.save_jsonl(eval_dev_result_list, "dev_nli_results.jsonl")
# Scoring
eval_mode = {'check_sent_id_correct': True, 'standard': True}
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
eval_dev_result_list,
common.load_jsonl(config.T_FEVER_DEV_JSONL),
mode=eval_mode,
verbose=False)
print("Fever Score(Strict/Acc./Precision/Recall/F1):", strict_score,
acc_score, pr, rec, f1)
print(f"Dev:{strict_score}/{acc_score}")
def train_fever_v1_advsample():
num_epoch = 12
seed = 12
batch_size = 32
lazy = True
dev_prob_threshold = 0.5
train_prob_threshold = 0.35
train_sample_top_k = 10
experiment_name = f"mesim_wn_450_adv_sample_v1_|t_prob:{train_prob_threshold}|top_k:{train_sample_top_k}"
print("Dev prob threshold:", dev_prob_threshold)
print("Train prob threshold:", train_prob_threshold)
print("Train sample top k:", train_sample_top_k)
dev_upstream_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/2018_07_20_15:17:59_r/dev_sent.jsonl")
train_upstream_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/2018_07_20_15:17:59_r/train_sent.jsonl")
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
p_dict = wn_persistent_api.persistence_load()
upstream_dev_list = score_converter_scaled(config.T_FEVER_DEV_JSONL,
dev_upstream_sent_list,
scale_prob=dev_prob_threshold,
delete_prob=False)
dev_fever_data_reader = WNReader(token_indexers=token_indexers,
lazy=lazy,
wn_p_dict=p_dict,
max_l=360)
train_fever_data_reader = WNReader(token_indexers=token_indexers,
lazy=lazy,
wn_p_dict=p_dict,
max_l=360)
complete_upstream_dev_data = get_actual_data(config.T_FEVER_DEV_JSONL,
upstream_dev_list)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=batch_size * 2)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden',
-2,
namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(
rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size,
1024 + 450),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300,
max_l=300)
print("Model Max length:", model.max_l)
model.display()
model.to(device)
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name),
'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
start_lr = 0.0002
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=start_lr)
criterion = nn.CrossEntropyLoss()
for i_epoch in range(num_epoch):
print("Resampling...")
# Resampling
# complete_upstream_train_data = get_sampled_data(config.T_FEVER_TRAIN_JSONL, train_upstream_file)
complete_upstream_train_data = get_adv_sampled_data(
config.T_FEVER_TRAIN_JSONL,
train_upstream_sent_list,
threshold_prob=train_prob_threshold,
top_n=train_sample_top_k)
print("Sample data length:", len(complete_upstream_train_data))
sampled_train_instances = train_fever_data_reader.read(
complete_upstream_train_data)
train_iter = biterator(sampled_train_instances,
shuffle=True,
num_epochs=1,
cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
y = batch['label']
loss = criterion(out, y)
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
iteration += 1
if i_epoch <= 6:
# mod = 5000
mod = 5000
else:
mod = 500
if iteration % mod == 0:
eval_iter = biterator(dev_instances,
shuffle=False,
num_epochs=1,
cuda_device=device_num)
dev_score, dev_loss = full_eval_model(
model, eval_iter, criterion, complete_upstream_dev_data)
print(f"Dev:{dev_score}/{dev_loss}")
need_save = False
if dev_score > best_dev:
best_dev = dev_score
need_save = True
if need_save:
save_path = os.path.join(
file_path_prefix,
f'i({iteration})_epoch({i_epoch})_dev({dev_score})_loss({dev_loss})_seed({seed})'
)
torch.save(model.state_dict(), save_path)
# Save some cache wordnet feature.
wn_persistent_api.persistence_update(p_dict)
def train_fever_std_ema_v1(resume_model=None, wn_feature=False):
"""
This method is the new training script for train fever with span and probability score.
:param resume_model:
:param wn_feature:
:return:
"""
num_epoch = 200
seed = 12
batch_size = 32
lazy = True
dev_prob_threshold = 0.1
train_prob_threshold = 0.1
train_sample_top_k = 8
experiment_name = f"nsmn_sent_wise_std_ema_lr1|t_prob:{train_prob_threshold}|top_k:{train_sample_top_k}"
# resume_model = None
print("Do EMA:")
print("Dev prob threshold:", dev_prob_threshold)
print("Train prob threshold:", train_prob_threshold)
print("Train sample top k:", train_sample_top_k)
dev_upstream_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/balanced_sentence_selection_results/dev_sent_pred_scores.jsonl"
)
train_upstream_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/balanced_sentence_selection_results/train_sent_scores.jsonl"
)
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
print("Building Prob Dicts...")
train_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/balanced_sentence_selection_results/train_sent_scores.jsonl"
)
dev_sent_list = common.load_jsonl(
config.RESULT_PATH /
"sent_retri_nn/balanced_sentence_selection_results/dev_sent_pred_scores.jsonl"
)
selection_dict = paired_selection_score_dict(train_sent_list)
selection_dict = paired_selection_score_dict(dev_sent_list, selection_dict)
upstream_dev_list = threshold_sampler_insure_unique(
config.T_FEVER_DEV_JSONL,
dev_upstream_sent_list,
prob_threshold=dev_prob_threshold,
top_n=5)
# Specifiy ablation to remove wordnet and number embeddings.
dev_fever_data_reader = WNSIMIReader(token_indexers=token_indexers,
lazy=lazy,
wn_p_dict=p_dict,
max_l=320,
ablation=None)
train_fever_data_reader = WNSIMIReader(token_indexers=token_indexers,
lazy=lazy,
wn_p_dict=p_dict,
max_l=320,
shuffle_sentences=False,
ablation=None)
complete_upstream_dev_data = select_sent_with_prob_for_eval(
config.T_FEVER_DEV_JSONL,
upstream_dev_list,
selection_dict,
tokenized=True)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden',
-2,
namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(
rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size,
1024 + 450 + dev_fever_data_reader.wn_feature_size),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300,
max_l=300,
use_extra_lex_feature=False,
max_span_l=100)
print("Model Max length:", model.max_l)
if resume_model is not None:
model.load_state_dict(torch.load(resume_model))
model.display()
model.to(device)
cloned_empty_model = copy.deepcopy(model)
ema: EMA = EMA(parameters=model.named_parameters())
# Create Log File
file_path_prefix, date = save_tool.gen_file_prefix(f"{experiment_name}")
# Save the source code.
script_name = os.path.basename(__file__)
with open(os.path.join(file_path_prefix, script_name),
'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Save source code end.
best_dev = -1
iteration = 0
start_lr = 0.0001
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=start_lr)
criterion = nn.CrossEntropyLoss()
for i_epoch in range(num_epoch):
print("Resampling...")
# Resampling
train_data_with_candidate_sample_list = \
threshold_sampler_insure_unique(config.T_FEVER_TRAIN_JSONL, train_upstream_sent_list,
train_prob_threshold,
top_n=train_sample_top_k)
complete_upstream_train_data = adv_simi_sample_with_prob_v1_1(
config.T_FEVER_TRAIN_JSONL,
train_data_with_candidate_sample_list,
selection_dict,
tokenized=True)
print("Sample data length:", len(complete_upstream_train_data))
sampled_train_instances = train_fever_data_reader.read(
complete_upstream_train_data)
train_iter = biterator(sampled_train_instances,
shuffle=True,
num_epochs=1,
cuda_device=device_num)
for i, batch in tqdm(enumerate(train_iter)):
model.train()
out = model(batch)
y = batch['label']
loss = criterion(out, y)
# No decay
optimizer.zero_grad()
loss.backward()
optimizer.step()
iteration += 1
# EMA update
ema(model.named_parameters())
if i_epoch < 15:
mod = 10000
# mod = 10
else:
mod = 2000
if iteration % mod == 0:
# eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
# complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
#
# eval_mode = {'check_sent_id_correct': True, 'standard': True}
# strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(complete_upstream_dev_data,
# common.load_jsonl(config.T_FEVER_DEV_JSONL),
# mode=eval_mode,
# verbose=False)
# print("Fever Score(Strict/Acc./Precision/Recall/F1):", strict_score, acc_score, pr, rec, f1)
#
# print(f"Dev:{strict_score}/{acc_score}")
# EMA saving
eval_iter = biterator(dev_instances,
shuffle=False,
num_epochs=1,
cuda_device=device_num)
load_ema_to_model(cloned_empty_model, ema)
complete_upstream_dev_data = hidden_eval(
cloned_empty_model, eval_iter, complete_upstream_dev_data)
eval_mode = {'check_sent_id_correct': True, 'standard': True}
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
complete_upstream_dev_data,
common.load_jsonl(config.T_FEVER_DEV_JSONL),
mode=eval_mode,
verbose=False)
print("Fever Score EMA(Strict/Acc./Precision/Recall/F1):",
strict_score, acc_score, pr, rec, f1)
print(f"Dev EMA:{strict_score}/{acc_score}")
need_save = False
if strict_score > best_dev:
best_dev = strict_score
need_save = True
if need_save:
# save_path = os.path.join(
# file_path_prefix,
# f'i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_seed({seed})'
# )
# torch.save(model.state_dict(), save_path)
ema_save_path = os.path.join(
file_path_prefix,
f'ema_i({iteration})_epoch({i_epoch})_dev({strict_score})_lacc({acc_score})_seed({seed})'
)
save_ema_to_file(ema, ema_save_path)
def eval_and_save_v2(model_path,
is_ema,
saving_dir,
save_train_data=True,
prob_thresholds=0.5):
# This method was modified on 21 NOV 2018
# for evaluating balanced trained selection model with different threshold value.
# It will then be used for later verification.
# Evaluate and Save all the sentence pairs results to be used for downstream verificaion
# 03 Oct 2018 03:56:40.
seed = 12
batch_size = 128
lazy = True
torch.manual_seed(seed)
keep_neg_sample_prob = 1
top_k_doc = 5
# sample_prob_decay = 0.05
dev_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/dev_doc.jsonl"
train_upstream_file = config.RESULT_PATH / "doc_retri/std_upstream_data_using_pageview/train_doc.jsonl"
# Prepare Data
token_indexers = {
'tokens':
SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(
namespace='elmo_characters') # This is the elmo_characters
}
train_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=lazy,
max_l=180)
dev_fever_data_reader = SSelectorReader(token_indexers=token_indexers,
lazy=lazy,
max_l=180)
complete_upstream_dev_data = get_full_list(config.T_FEVER_DEV_JSONL,
dev_upstream_file,
pred=True,
top_k=top_k_doc)
complete_upstream_train_data = get_full_list(config.T_FEVER_TRAIN_JSONL,
train_upstream_file,
pred=False,
top_k=top_k_doc)
print("Dev size:", len(complete_upstream_dev_data))
print("Train size:", len(complete_upstream_train_data))
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
train_instances = train_fever_data_reader.read(
complete_upstream_train_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
dev_biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT /
"vocab_cache" / "nli_basic")
# THis is important
vocab.add_token_to_namespace("true", namespace="selection_labels")
vocab.add_token_to_namespace("false", namespace="selection_labels")
vocab.add_token_to_namespace("hidden", namespace="selection_labels")
vocab.change_token_with_index_to_namespace("hidden",
-2,
namespace='selection_labels')
# Label value
vocab.get_index_to_token_vocabulary('selection_labels')
print(vocab.get_token_to_index_vocabulary('selection_labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
dev_biterator.index_with(vocab)
# exit(0)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu",
index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
embedding_dim=300,
max_l=160,
num_of_class=2)
if not is_ema:
model.load_state_dict(torch.load(model_path))
else:
load_ema_to_model(model, model_path)
model.display()
model.to(device)
dev_actual_list = common.load_jsonl(config.T_FEVER_DEV_JSONL)
train_actual_list = common.load_jsonl(config.T_FEVER_TRAIN_JSONL)
eval_iter = dev_biterator(dev_instances, shuffle=False, num_epochs=1)
train_iter = biterator(train_instances, shuffle=False, num_epochs=1)
complete_upstream_dev_data = hidden_eval(model, eval_iter,
complete_upstream_dev_data)
if save_train_data:
complete_upstream_train_data = hidden_eval(
model, train_iter, complete_upstream_train_data)
common.save_jsonl(complete_upstream_train_data,
Path(str(saving_dir)) / "train_sent_scores.jsonl")
common.save_jsonl(complete_upstream_dev_data,
Path(str(saving_dir)) / "dev_sent_pred_scores.jsonl")
if not isinstance(prob_thresholds, list):
prob_thresholds = [prob_thresholds]
for scal_prob in prob_thresholds:
print("Eval Dev Data prob_threshold:", scal_prob)
dev_results_list = score_converter_v1(config.T_FEVER_DEV_JSONL,
complete_upstream_dev_data,
sent_retri_top_k=5,
sent_retri_scal_prob=scal_prob)
# This is only a wrapper for the simi_sampler
eval_mode = {'check_sent_id_correct': True, 'standard': True}
for a, b in zip(dev_actual_list, dev_results_list):
b['predicted_label'] = a['label']
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
dev_results_list, dev_actual_list, mode=eval_mode, verbose=False)
tracking_score = strict_score
print(f"Dev(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/")
print("Strict score:", strict_score)
print(f"Eval Tracking score:", f"{tracking_score}")
if save_train_data:
print("Build Train Data")
train_results_list = score_converter_v1(
config.T_FEVER_TRAIN_JSONL,
complete_upstream_train_data,
sent_retri_top_k=5,
sent_retri_scal_prob=prob_threshold)
# This is only a wrapper for the simi_sampler
eval_mode = {'check_sent_id_correct': True, 'standard': True}
for a, b in zip(train_actual_list, train_results_list):
b['predicted_label'] = a['label']
strict_score, acc_score, pr, rec, f1 = c_scorer.fever_score(
train_results_list,
train_actual_list,
mode=eval_mode,
verbose=False)
tracking_score = strict_score
print(f"Train(raw_acc/pr/rec/f1):{acc_score}/{pr}/{rec}/{f1}/")
print("Strict score:", strict_score)
print(f"Eval Tracking score:", f"{tracking_score}")
build_vocab_embeddings(vocab, config.DATA_ROOT / "embeddings/glove.840B.300d.txt",
embd_dim=300, saved_path=config.DATA_ROOT / "vocab_cache" / "nli_basic")
if __name__ == '__main__':
# nli_path_list = []
# for in_file in (config.DATA_ROOT / "mnli").iterdir():
# nli_path_list.append(in_file)
#
# for in_file in (config.DATA_ROOT / "snli").iterdir():
# nli_path_list.append(in_file)
#
# build_fever_vocab_with_embeddings_and_save()
# build_vocab_embeddings(vocab, config.DATA_ROOT / "embeddings/glove.840B.300d.txt",
# embd_dim=300, saved_path=config.DATA_ROOT / "vocab_cache" / "nli")
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
print(weight_dict)
print(vocab)
# print(vocab.get_vocab_size())
# vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
#
# print(vocab.get_token_to_index_vocabulary('labels'))
# print(vocab.get_index_to_token_vocabulary('labels'))
# print()
# print(weight_dict['glove.840B.300d'].size())
#
def pipeline_nli_run(t_org_file, upstream_dev_data_list, upstream_sent_file_list, model_path):
batch_size = 32
lazy = True
print("Size:", len(upstream_dev_data_list))
print("Building Prob Dicts...")
selection_dict = dict()
for upstream_sent_file in upstream_sent_file_list:
upstream_sent_l = common.load_jsonl(upstream_sent_file)
selection_dict = paired_selection_score_dict(upstream_sent_l, selection_dict)
# Prepare Data
token_indexers = {
'tokens': SingleIdTokenIndexer(namespace='tokens'), # This is the raw tokens
'elmo_chars': ELMoTokenCharactersIndexer(namespace='elmo_characters') # This is the elmo_characters
}
p_dict = wn_persistent_api.persistence_load()
dev_fever_data_reader = WNSIMIReader(token_indexers=token_indexers, lazy=lazy, wn_p_dict=p_dict, max_l=360)
complete_upstream_dev_data = select_sent_with_prob_for_eval(t_org_file, upstream_dev_data_list,
selection_dict, tokenized=True, pipeline=True)
complete_upstream_dev_data = append_hidden_label(complete_upstream_dev_data)
dev_instances = dev_fever_data_reader.read(complete_upstream_dev_data)
# Load Vocabulary
biterator = BasicIterator(batch_size=batch_size)
vocab, weight_dict = load_vocab_embeddings(config.DATA_ROOT / "vocab_cache" / "nli_basic")
vocab.change_token_with_index_to_namespace('hidden', -2, namespace='labels')
print(vocab.get_token_to_index_vocabulary('labels'))
print(vocab.get_vocab_size('tokens'))
biterator.index_with(vocab)
# Build Model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu", index=0)
device_num = -1 if device.type == 'cpu' else 0
model = Model(rnn_size_in=(1024 + 300 + dev_fever_data_reader.wn_feature_size,
1024 + 450),
rnn_size_out=(450, 450),
weight=weight_dict['glove.840B.300d'],
vocab_size=vocab.get_vocab_size('tokens'),
mlp_d=900,
embedding_dim=300, max_l=300)
print("Model Max length:", model.max_l)
model.load_state_dict(torch.load(model_path))
model.display()
model.to(device)
eval_iter = biterator(dev_instances, shuffle=False, num_epochs=1, cuda_device=device_num)
complete_upstream_dev_data = hidden_eval(model, eval_iter, complete_upstream_dev_data)
wn_persistent_api.persistence_update(p_dict)
return complete_upstream_dev_data
